From mhammond at users.sourceforge.net Mon Sep 1 00:04:10 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 1 02:04:18 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/sandbox dump_props.py,
1.9, 1.10
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/sandbox
In directory sc8-pr-cvs1:/tmp/cvs-serv9969
Modified Files:
dump_props.py
Log Message:
Extract the HTML from the RTF, just like the plugin.
Index: dump_props.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/sandbox/dump_props.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** dump_props.py 29 Jul 2003 00:35:24 -0000 1.9
--- dump_props.py 1 Sep 2003 06:04:08 -0000 1.10
***************
*** 47,50 ****
--- 47,58 ----
all_props.sort() # sort by first tuple item, which is name :)
for prop_name, prop_tag, prop_val in all_props:
+ # Do some magic rtf conversion
+ if PROP_ID(prop_tag) == PROP_ID(PR_RTF_COMPRESSED):
+ rtf_stream = item.OpenProperty(PR_RTF_COMPRESSED, pythoncom.IID_IStream,
+ 0, 0)
+ html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0)
+ prop_val = mapi.RTFStreamToHTML(html_stream)
+ prop_name = "PR_RTF_COMPRESSED (to HTML)"
+ prop_tag = PROP_TAG(PT_STRING8, PR_RTF_COMPRESSED)
if get_large_props and \
PROP_TYPE(prop_tag)==PT_ERROR and \
From mhammond at users.sourceforge.net Mon Sep 1 00:04:37 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 1 02:04:40 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/sandbox extract_prop.py,
NONE, 1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/sandbox
In directory sc8-pr-cvs1:/tmp/cvs-serv10074
Added Files:
extract_prop.py
Log Message:
Tool to extract a binary property to a file.
--- NEW FILE: extract_prop.py ---
# Extract a property from a message to a file. This is about the only
# way to extract huge properties such that the original data is available.
import sys, os
import mapi_driver
from win32com.mapi import mapitags, mapi
import pythoncom
def DumpItemProp(item, prop, outfile):
if type(prop)!=type(0):
# see if a mapitags contant
try:
prop = mapitags.__dict__[prop]
except KeyError:
# resolve as a name
props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
propIds = obj.GetIDsFromNames(props, 0)
prop = mapitags.PROP_TAG( mapitags.PT_UNSPECIFIED, mapitags.PROP_ID(propIds[0]))
hr, data = item.GetProps((prop,), 0)
prop_tag, prop_val = data[0]
# Do some magic rtf conversion
if mapitags.PROP_ID(prop_tag) == mapitags.PROP_ID(mapitags.PR_RTF_COMPRESSED):
rtf_stream = item.OpenProperty(mapitags.PR_RTF_COMPRESSED, pythoncom.IID_IStream,
0, 0)
html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0)
chunks = []
while 1:
chunk = html_stream.Read(4096)
if not chunk:
break
chunks.append(chunk)
prop_val = "".join(chunks)
elif mapitags.PROP_TYPE(prop_tag)==mapitags.PT_ERROR and \
prop_val in [mapi.MAPI_E_NOT_ENOUGH_MEMORY,'MAPI_E_NOT_ENOUGH_MEMORY']:
prop_tag = mapitags.PROP_TAG(mapitags.PT_BINARY, mapitags.PROP_ID(prop_tag))
stream = item.OpenProperty(prop_tag,
pythoncom.IID_IStream,
0, 0)
chunks = []
while 1:
chunk = stream.Read(4096)
if not chunk:
break
chunks.append(chunk)
prop_val = "".join(chunks)
outfile.write(prop_val)
def DumpProp(driver, mapi_folder, subject, prop_tag, outfile):
hr, data = mapi_folder.GetProps( (mapitags.PR_DISPLAY_NAME_A,), 0)
name = data[0][1]
items = driver.GetItemsWithValue(mapi_folder, mapitags.PR_SUBJECT_A, subject)
num = 0
for item in items:
if num > 1:
print >> sys.stderr, "Warning: More than one matching item - ignoring"
break
DumpItemProp(item, prop_tag, outfile)
num += 1
if num==0:
print >> sys.stderr, "Error: No matching items"
def usage(driver):
folder_doc = driver.GetFolderNameDoc()
msg = """\
Usage: %s [-f foldername] [-o output_file] -p property_name subject of the message
-f - Search for the message in the specified folder (default = Inbox)
-p - Name of the property to dump
-o - Output file to be created - default - stdout.
Dumps all properties for all messages that match the subject. Subject
matching is substring and ignore-case.
%s
Use the -n option to see all top-level folder names from all stores.""" \
% (os.path.basename(sys.argv[0]),folder_doc)
print msg
def main():
driver = mapi_driver.MAPIDriver()
import getopt
try:
opts, args = getopt.getopt(sys.argv[1:], "np:f:o:")
except getopt.error, e:
print e
print
usage(driver)
sys.exit(1)
folder_name = prop_name = output_name = ""
for opt, opt_val in opts:
if opt == "-p":
prop_name = opt_val
elif opt == "-f":
folder_name = opt_val
elif opt == '-o':
output_name = os.path.abspath(opt_val)
elif opt == "-n":
driver.DumpTopLevelFolders()
sys.exit(1)
else:
print "Invalid arg"
return
if not folder_name:
folder_name = "Inbox" # Assume this exists!
subject = " ".join(args)
if not subject:
print "You must specify a subject"
print
usage(driver)
sys.exit(1)
if not prop_name:
print "You must specify a property"
print
usage(driver)
sys.exit(1)
if output_name:
output_file = file(output_name, "wb")
else:
output_file = sys.stdout
try:
folder = driver.FindFolder(folder_name)
except ValueError, details:
print details
sys.exit(1)
DumpProp(driver, folder, subject, prop_name, output_file)
if __name__=='__main__':
main()
From richiehindle at users.sourceforge.net Mon Sep 1 00:07:29 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Mon Sep 1 02:07:33 2003
Subject: [Spambayes-checkins] spambayes/spambayes Dibbler.py, 1.5,
1.6 Options.py, 1.68, 1.69 ProxyUI.py, 1.19,
1.20 UserInterface.py, 1.20, 1.21
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10484/spambayes
Modified Files:
Dibbler.py Options.py ProxyUI.py UserInterface.py
Log Message:
HTTP-Auth support for the web interface (many thanks to Romain Guy).
Index: Dibbler.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Dibbler.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** Dibbler.py 31 Aug 2003 02:09:40 -0000 1.5
--- Dibbler.py 1 Sep 2003 06:07:27 -0000 1.6
***************
*** 170,174 ****
import StringIO
! import os, sys, re, time, traceback
import socket, asyncore, asynchat, cgi, urlparse, webbrowser
--- 170,174 ----
import StringIO
! import os, sys, re, time, traceback, md5, base64
import socket, asyncore, asynchat, cgi, urlparse, webbrowser
***************
*** 291,294 ****
--- 291,298 ----
"""
+ NO_AUTHENTICATION = "None"
+ BASIC_AUTHENTICATION = "Basic"
+ DIGEST_AUTHENTICATION = "Digest"
+
def __init__(self, port=('', 80), context=_defaultContext):
"""Create an `HTTPServer` for the given port."""
***************
*** 307,310 ****
--- 311,338 ----
self._plugins.append(plugin)
+ def requestAuthenticationMode(self):
+ """Override: HTTP Authentication. It should return a value among
+ NO_AUTHENTICATION, BASIC_AUTHENTICATION and DIGEST_AUTHENTICATION.
+ The two last values will force HTTP authentication respectively
+ through Base64 and MD5 encodings."""
+ return self.NO_AUTHENTICATION
+
+ def isValidUser(self, name, password):
+ """Override: Return True for authorized logins."""
+ return True
+
+ def getPasswordForUser(self, name):
+ """Override: Return the password associated to the specified user
+ name."""
+ return ''
+
+ def getRealm(self):
+ """Override: Specify the HTTP authentication realm."""
+ return "Dibbler application server"
+
+ def getCancelMessage(self):
+ """Override: Specify the cancel message for an HTTP Authentication."""
+ return "You must log in."""
+
class _HTTPHandler(BrighterAsyncChat):
***************
*** 385,388 ****
--- 413,442 ----
params[name] = value[0]
+ # Parse the headers.
+ headersRegex = re.compile('([^:]*):\s*(.*)')
+ headersDict = dict([headersRegex.match(line).groups(2)
+ for line in headers.split('\r\n')
+ if headersRegex.match(line)])
+
+ # HTTP Basic/Digest Authentication support.
+ serverAuthMode = self._server.requestAuthenticationMode()
+ if serverAuthMode != HTTPServer.NO_AUTHENTICATION:
+ # The server wants us to authenticate the user.
+ authResult = False
+ authHeader = headersDict.get('Authorization')
+ if authHeader:
+ authMatch = re.search('(\w+)\s+(.*)', authHeader)
+ authenticationMode, login = authMatch.groups()
+
+ if authenticationMode == HTTPServer.BASIC_AUTHENTICATION:
+ authResult = self._basicAuthentication(login)
+ elif authenticationMode == HTTPServer.DIGEST_AUTHENTICATION:
+ authResult = self._digestAuthentication(login, method)
+ else:
+ print >>sys.stdout, "Unknown mode: %s" % authenticationMode
+
+ if not authResult:
+ self.writeUnauthorizedAccess(serverAuthMode)
+
# Find and call the methlet. '/eggs.gif' becomes 'onEggsGif'.
if path == '/':
***************
*** 492,495 ****
--- 546,637 ----
content = ''
self.push('\r\n'.join(headers) + str(content))
+
+ def writeUnauthorizedAccess(self, authenticationMode):
+ """Access is protected by HTTP authentication."""
+ if authenticationMode == HTTPServer.BASIC_AUTHENTICATION:
+ authString = self._getBasicAuthString()
+ elif authenticationMode == HTTPServer.DIGEST_AUTHENTICATION:
+ authString = self._getDigestAuthString()
+ else:
+ self.writeError(500, "Inconsistent authentication mode.")
+ return
+
+ headers = []
+ headers.append('HTTP/1.0 401 Unauthorized')
+ headers.append('WWW-Authenticate: ' + authString)
+ headers.append('Connection: close')
+ headers.append('Content-Type: text/html')
+ headers.append('')
+ headers.append('')
+ self.write('\r\n'.join(headers) + self._server.getCancelMessage())
+ self.close_when_done()
+
+ def _getDigestAuthString(self):
+ """Builds the WWW-Authenticate header for Digest authentication."""
+ authString = 'Digest realm="' + self._server.getRealm() + '"'
+ authString += ', nonce="' + self._getCurrentNonce() + '"'
+ authString += ', opaque="0000000000000000"'
+ authString += ', stale="false"'
+ authString += ', algorithm="MD5"'
+ authString += ', qop="auth"'
+ return authString
+
+ def _getBasicAuthString(self):
+ """Builds the WWW-Authenticate header for Basic authentication."""
+ return 'Basic realm="' + self._server.getRealm() + '"'
+
+ def _getCurrentNonce(self):
+ """Returns the current nonce value. This value is a Base64 encoding
+ of current time plus one minute. This means the nonce will expire a
+ minute from now."""
+ timeString = time.asctime(time.localtime(time.time() + 60))
+ return base64.encodestring(timeString).rstrip('\n=')
+
+ def _isValidNonce(self, nonce):
+ """Check if the specified nonce is still valid. A nonce is invalid
+ when its time converted value is lower than current time."""
+ padAmount = len(nonce) % 4
+ if padAmount > 0: padAmount = 4 - padAmount
+ nonce += '=' * (len(nonce) + padAmount)
+
+ decoded = base64.decodestring(nonce)
+ return time.time() < time.mktime(time.strptime(decoded))
+
+ def _basicAuthentication(self, login):
+ """Performs a Basic HTTP authentication. Returns True when the user
+ has logged in successfully, False otherwise."""
+ userName, password = base64.decodestring(login).split(':')
+ return self._server.isValidUser(userName, password)
+
+ def _digestAuthentication(self, login, method):
+ """Performs a Digest HTTP authentication. Returns True when the user
+ has logged in successfully, False otherwise."""
+ def stripQuotes(s):
+ return (s[0] == '"' and s[-1] == '"') and s[1:-1] or s
+
+ options = dict([s.split('=') for s in login.split(", ")])
+ userName = stripQuotes(options["username"])
+ password = self._server.getPasswordForUser(userName)
+ nonce = stripQuotes(options["nonce"])
+
+ # The following computations are based upon RFC 2617.
+ A1 = "%s:%s:%s" % (userName, self._server.getRealm(), password)
+ HA1 = md5.new(A1).hexdigest()
+ A2 = "%s:%s" % (method, stripQuotes(options["uri"]))
+ HA2 = md5.new(A2).hexdigest()
+
+ unhashedDigest = ""
+ if options.has_key("qop"):
+ unhashedDigest = "%s:%s:%s:%s:%s:%s" % \
+ (HA1, nonce,
+ stripQuotes(options["nc"]),
+ stripQuotes(options["cnonce"]),
+ stripQuotes(options["qop"]), HA2)
+ else:
+ unhashedDigest = "%s:%s:%s" % (HA1, nonce, HA2)
+ hashedDigest = md5.new(unhashedDigest).hexdigest()
+
+ return (stripQuotes(options["response"]) == hashedDigest and
+ self._isValidNonce(nonce))
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.68
retrieving revision 1.69
diff -C2 -d -r1.68 -r1.69
*** Options.py 28 Aug 2003 21:15:06 -0000 1.68
--- Options.py 1 Sep 2003 06:07:27 -0000 1.69
***************
*** 832,835 ****
--- 832,853 ----
want to quickly identify mail received via a mailing list.""",
BOOLEAN, RESTORE),
+
+ ("http_authentication", "HTTP Authentication", "None",
+ """This option lets you choose the security level of the web interface.
+ When selecting Basic or Digest, the user will be prompted a login and a
+ password to access the web interface. The Basic option is faster, but
+ transmits the password in clear on the network. The Digest option
+ encrypts the password before transmission.""",
+ ("None", "Basic", "Digest"), RESTORE),
+
+ ("http_user_name", "User name", "admin",
+ """If you activated the HTTP authentication option, you can modify the
+ authorized user name here.""",
+ r"[\w]+", RESTORE),
+
+ ("http_password", "Password", "admin",
+ """If you activated the HTTP authentication option, you can modify the
+ authorized user password here.""",
+ r"[\w]+", RESTORE),
),
Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** ProxyUI.py 26 Aug 2003 04:30:41 -0000 1.19
--- ProxyUI.py 1 Sep 2003 06:07:27 -0000 1.20
***************
*** 85,88 ****
--- 85,91 ----
('html_ui', 'display_to'),
('html_ui', 'allow_remote_connections'),
+ ('html_ui', 'http_authentication'),
+ ('html_ui', 'http_user_name'),
+ ('html_ui', 'http_password'),
('Header Options', None),
('pop3proxy', 'notate_to'),
Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** UserInterface.py 30 Aug 2003 21:35:16 -0000 1.20
--- UserInterface.py 1 Sep 2003 06:07:27 -0000 1.21
***************
*** 95,98 ****
--- 95,115 ----
print 'User interface url is http://localhost:%d/' % (uiPort)
+ def requestAuthenticationMode(self):
+ return options["html_ui", "http_authentication"]
+
+ def getRealm(self):
+ return "SpamBayes Web Interface"
+
+ def isValidUser(self, name, password):
+ return (name == options["html_ui", "http_user_name"] and
+ password == options["html_ui", "http_password"])
+
+ def getPasswordForUser(self, name):
+ # There is only one login available in the web interface.
+ return options["html_ui", "http_password"]
+
+ def getCancelMessage(self):
+ return "You must login to use SpamBayes."""
+
class BaseUserInterface(Dibbler.HTTPPlugin):
From mhammond at users.sourceforge.net Mon Sep 1 01:27:33 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 1 03:27:42 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.21,1.22
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv21197
Modified Files:
about.html
Log Message:
Worm around Mozilla bug. Add full-stop and reword bit about Outlook rules.
Index: about.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** about.html 29 Aug 2003 15:18:22 -0000 1.21
--- about.html 1 Sep 2003 07:27:31 -0000 1.22
***************
*** 6,13 ****
! |
!   |
! |
SpamBayes Outlook Plugin
--- 6,19 ----
!
!
! |
! |
! |
!
!
SpamBayes Outlook Plugin
***************
*** 26,30 ****
Other links of interest
The main SpamBayes project page.
! The online SpamBayes FAQ
How to make a
donation.
--- 32,36 ----
Other links of interest
The main SpamBayes project page.
! The online SpamBayes FAQ.
How to make a
donation.
***************
*** 35,41 ****
When Outlook is under load, SpamBayes may occasionally miss some
messages (as Outlook doesn't tell us they appeared). Also, if you
! use builtin Outlook rules, you may find that occasionally they fails to
! filter messages they normally would (as both SpamBayes and the builtin
! rules "fight" over processing the message). Both of these
problems can generally be solved by enabling background filtering of
messages, via the SpamBayes Manager's Advanced
--- 41,47 ----
When Outlook is under load, SpamBayes may occasionally miss some
messages (as Outlook doesn't tell us they appeared). Also, if you
! use builtin Outlook rules, you may find that occasionally they also
! fail to filter messages (as both SpamBayes and the builtin rules
! "fight" over processing the message). Both of these
problems can generally be solved by enabling background filtering of
messages, via the SpamBayes Manager's Advanced
From mhammond at users.sourceforge.net Mon Sep 1 01:29:50 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 1 03:29:54 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.70,1.71
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv21408
Modified Files:
msgstore.py
Log Message:
Handle MAPI failures fetching the message when trying to get a field value.
New RTF->HTML code - however, it won't actually extract HTML until you
upgrade your win32all to a version that is yet to be released.
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.70
retrieving revision 1.71
diff -C2 -d -r1.70 -r1.71
*** msgstore.py 31 Aug 2003 05:38:52 -0000 1.70
--- msgstore.py 1 Sep 2003 07:29:48 -0000 1.71
***************
*** 424,427 ****
--- 424,454 ----
return ret
+ # Some nasty stuff for getting RTF out of the message
+ _have_complained_about_missing_rtf = False
+ def GetHTMLFromRTFProperty(mapi_object, prop_tag = PR_RTF_COMPRESSED):
+ global _have_complained_about_missing_rtf
+ try:
+ rtf_stream = mapi_object.OpenProperty(prop_tag, pythoncom.IID_IStream,
+ 0, 0)
+ except pythoncom.com_error, details:
+ if not IsNotFoundCOMException(details):
+ print "ERROR getting RTF body", details
+ return ""
+ try:
+ html_stream = mapi.WrapCompressedRTFStream(rtf_stream, 0)
+ except AttributeError:
+ if not _have_complained_about_missing_rtf:
+ print "*" * 50
+ print "Sorry, but you need to update to a new win32all (158 or "
+ print "later), so we correctly get the HTML from messages."
+ print "See http://starship.python.net/crew/mhammond/win32"
+ print "*" * 50
+ _have_complained_about_missing_rtf = True
+ return ""
+ html = mapi.RTFStreamToHTML(html_stream)
+ # html may be None if not RTF originally from HTML, but here we
+ # always want a string
+ return html or ''
+
class MAPIMsgStoreFolder(MsgStoreMsg):
def __init__(self, msgstore, id, name, count):
***************
*** 705,709 ****
def _GetMessageTextParts(self):
! # This is finally reliable. The only messages this now fails for
# are for "forwarded" messages, where the forwards are actually
# in an attachment. Later.
--- 732,736 ----
def _GetMessageTextParts(self):
! # This is almost reliable :). The only messages this now fails for
# are for "forwarded" messages, where the forwards are actually
# in an attachment. Later.
***************
*** 720,723 ****
--- 747,753 ----
html = self._GetPotentiallyLargeStringProp(prop_ids[1], data[1])
headers = self._GetPotentiallyLargeStringProp(prop_ids[2], data[2])
+ # xxx - not sure what to do if we have both.
+ if not html:
+ html = GetHTMLFromRTFProperty(self.mapi_object)
# Some Outlooks deliver a strange notion of headers, including
***************
*** 937,941 ****
def GetField(self, prop, raise_errors = False):
! self._EnsureObject()
if type(prop) != type(0):
props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
--- 967,977 ----
def GetField(self, prop, raise_errors = False):
! try:
! self._EnsureObject()
! except pythoncom.com_error, details:
! if not IsNotFoundCOMException(details):
! print "ERROR: Could not open an object to fetch a field"
! print details
! return None
if type(prop) != type(0):
props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
From anadelonbrin at users.sourceforge.net Mon Sep 1 03:05:54 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 05:06:02 2003
Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.21,
1.22 classifier.py, 1.6, 1.7 storage.py, 1.24, 1.25
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv4810/spambayes
Modified Files:
UserInterface.py classifier.py storage.py
Log Message:
Add the ability to get a list of the words (i.e. keys) in a classifier (SQL versions
still coming...)
Add [ 796832 ] Word query should show all words starting with certain text
(which essentially is the ability to do a wildcard search through the classifier
database through the web interface).
Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** UserInterface.py 1 Sep 2003 06:07:27 -0000 1.21
--- UserInterface.py 1 Sep 2003 09:05:51 -0000 1.22
***************
*** 281,304 ****
def onWordquery(self, word):
if word == "":
stats = "You must enter a word."
else:
word = word.lower()
! wordinfo = classifier._wordinfoget(word)
! if wordinfo:
! stats = self.html.wordStats.clone()
! stats.spamcount = wordinfo.spamcount
! stats.hamcount = wordinfo.hamcount
! stats.spamprob = classifier.probability(wordinfo)
else:
! stats = "%r does not exist in the database." % cgi.escape(word)
query = self.html.wordQuery.clone()
! query.word.value = word
! statsBox = self._buildBox("Statistics for %r" % cgi.escape(word),
! 'status.gif', stats)
queryBox = self._buildBox("Word query", 'query.gif', query)
self._writePreamble("Word query")
! self.write(statsBox + queryBox)
self._writePostamble()
--- 281,356 ----
def onWordquery(self, word):
+ wildcard_limit = 10
+ statsBoxes = []
if word == "":
stats = "You must enter a word."
+ statsBoxes.append(self._buildBox("Statistics for %r" % \
+ cgi.escape(word),
+ 'status.gif', stats))
else:
word = word.lower()
! if word[-1] == '*':
! # Wildcard search - list all words that start with word[:-1]
! word = word[:-1]
! reached_limit = False
! for w in classifier._wordinfokeys():
! if not reached_limit and len(statsBoxes) > wildcard_limit:
! reached_limit = True
! over_limit = 0
! if w.startswith(word):
! if reached_limit:
! over_limit += 1
! else:
! wordinfo = classifier._wordinfoget(w)
! stats = self.html.wordStats.clone()
! stats.spamcount = wordinfo.spamcount
! stats.hamcount = wordinfo.hamcount
! stats.spamprob = classifier.probability(wordinfo)
! box = self._buildBox("Statistics for %r" % \
! cgi.escape(w),
! 'status.gif', stats)
! statsBoxes.append(box)
! if len(statsBoxes) == 0:
! stats = "There are no words that begin with '%s' " \
! "in the database." % (word,)
! # We build a box for each word; I'm not sure this is
! # produces the nicest results, but it's ok with a
! # limited number of words.
! statsBoxes.append(self._buildBox("Statistics for %s" % \
! cgi.escape(word),
! 'status.gif', stats))
! elif reached_limit:
! if over_limit == 1:
! singles = ["was", "match", "is"]
! else:
! singles = ["were", "matches", "are"]
! stats = "There %s %d additional %s that %s not " \
! "shown here." % (singles[0], over_limit,
! singles[1], singles[2])
! box = self._buildBox("Statistics for '%s*'" % \
! cgi.escape(word), 'status.gif',
! stats)
! statsBoxes.append(box)
else:
! # Optimised version for non-wildcard searches
! wordinfo = classifier._wordinfoget(word)
! if wordinfo:
! stats = self.html.wordStats.clone()
! stats.spamcount = wordinfo.spamcount
! stats.hamcount = wordinfo.hamcount
! stats.spamprob = classifier.probability(wordinfo)
! else:
! stats = "%r does not exist in the database." % cgi.escape(word)
! statsBoxes.append(self._buildBox("Statistics for %r" % \
! cgi.escape(word),
! 'status.gif', stats))
query = self.html.wordQuery.clone()
! query.word.value = "%s" % (word,)
queryBox = self._buildBox("Word query", 'query.gif', query)
self._writePreamble("Word query")
! for box in statsBoxes:
! self.write(box)
! self.write(queryBox)
self._writePostamble()
Index: classifier.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** classifier.py 26 May 2003 04:55:40 -0000 1.6
--- classifier.py 1 Sep 2003 09:05:51 -0000 1.7
***************
*** 469,472 ****
--- 469,475 ----
del self.wordinfo[word]
+ def _wordinfokeys(self):
+ return self.wordinfo.keys()
+
Bayes = Classifier
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** storage.py 31 Aug 2003 21:10:58 -0000 1.24
--- storage.py 1 Sep 2003 09:05:51 -0000 1.25
***************
*** 269,272 ****
--- 269,277 ----
self.changed_words[word] = WORD_DELETED
+ def _wordinfokeys(self):
+ wordinfokeys = self.db.keys()
+ del wordinfokeys[wordinfokeys.index(self.statekey)]
+ return wordinfokeys
+
class SQLClassifier(classifier.Classifier):
From anadelonbrin at users.sourceforge.net Mon Sep 1 03:16:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 05:16:16 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.14,
1.15 WHAT_IS_NEW.txt, 1.11, 1.12
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6042
Modified Files:
CHANGELOG.txt WHAT_IS_NEW.txt
Log Message:
Bring up to date.
Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** CHANGELOG.txt 1 Sep 2003 05:47:04 -0000 1.14
--- CHANGELOG.txt 1 Sep 2003 09:16:08 -0000 1.15
***************
*** 1,4 ****
--- 1,6 ----
Alpha Release 5
===============
+ Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319.
+ Tony Meyer 01/09/2003 Added [ 796832 ] Word query should show all words starting with certain text
Tony Meyer 01/09/2003 Fix for [ 797316 ] Extra CRLF to smtp server causes garbage error.
Richie Hindle 31/08/2003 The web UI's Shutdown command, and stopping the pop3proxy_service, now wait for any open proxy connections to finish before exiting the process.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** WHAT_IS_NEW.txt 1 Sep 2003 05:47:04 -0000 1.11
--- WHAT_IS_NEW.txt 1 Sep 2003 09:16:08 -0000 1.12
***************
*** 93,96 ****
--- 93,101 ----
o Prevent the "Show clues" links on the web interface's training page
from word-wrapping and making all the table rows two lines high.
+ o You can now put "*" at the end of a word in the "Word Query" box
+ on the web interface, and have it show you the first ten words,
+ and how many words there are in total, in the database that start with
+ that word.
+ o The web interface now supports HTTP-Auth.
o Added a new script (code-named 'overkill.py') which enables
'drag and drop' training for POP3 users. This is currently still in
***************
*** 155,159 ****
The following feature requests tracked via the Sourceforge system were
added:
! 789916, 698036
A url containing the details of these feature requests can be made by
--- 160,164 ----
The following feature requests tracked via the Sourceforge system were
added:
! 789916, 698036, 796832, 791319
A url containing the details of these feature requests can be made by
***************
*** 165,169 ****
===================
The following patches tracked via the Sourceforge system were integrated:
! 791254, 790615, 788001, 769981
A url containing the details of these feature requests can be made by
--- 170,174 ----
===================
The following patches tracked via the Sourceforge system were integrated:
! 791254, 790615, 788001, 769981, 791393
A url containing the details of these feature requests can be made by
From anadelonbrin at users.sourceforge.net Mon Sep 1 03:39:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 05:39:39 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.25,1.26
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9393/spambayes
Modified Files:
storage.py
Log Message:
Add the ability to get a list of the words (i.e. keys) in a SQL based classifier.
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** storage.py 1 Sep 2003 09:05:51 -0000 1.25
--- storage.py 1 Sep 2003 09:39:31 -0000 1.26
***************
*** 377,380 ****
--- 377,392 ----
self._delete_row(word)
+ def _wordinfokeys(self):
+ c = self.cursor()
+ c.execute("select word from bayes")
+ rows = self.fetchall(c)
+ # There is probably some clever way to do this with map or
+ # something, but I don't know what it is. We want the first
+ # element from all the items in 'rows'
+ keys = []
+ for r in rows:
+ keys.append(r[0])
+ return keys
+
class PGClassifier(SQLClassifier):
From anadelonbrin at users.sourceforge.net Mon Sep 1 04:05:00 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 06:07:39 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,NONE,1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv13593/windows
Added Files:
pop3proxy_tray.py
Log Message:
A basic shell for a pop3proxy tray application, which can open up the configuration
page, and start and stop the proxy. The start/stop code needs a lot more work,
as part of the pop3proxy binary process.
Note that this will cause Python 2.3 to unexpectedly terminate, but will work fine
in Python 2.2. See [ 798452 ] win32gui_taskbar.py broken in Python 2.3
--- NEW FILE: pop3proxy_tray.py ---
#!/usr/bin/env python
"""A script to provide an icon in the Windows taskbar tray to control the
POP3 proxy.
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "Mark Hammond, all the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
# Heavily based on the win32gui_taskbar.py demo from Mark Hammond's
# win32 extensions.
import os
import sys
import webbrowser
import win32con
from win32api import *
from win32gui import *
# Allow for those without SpamBayes on the PYTHONPATH
sys.path.insert(-1, os.getcwd())
sys.path.insert(-1, os.path.dirname(os.getcwd()))
import pop3proxy
from spambayes import Dibbler
from spambayes.Options import options
class MainWindow(object):
def __init__(self):
# The ordering here is important - it is the order that they will
# appear in the menu. As dicts don't have an order, this means
# that the order is controlled by the id. Any items were the
# function is None will appear as separators.
self.control_functions = {1024 : ("Start SpamBayes", self.StartStop),
1025 : ("-", None),
1026 : ("View information ...", self.OpenInterface),
1027 : ("Configure ...", self.OpenConfig),
1028 : ("-", None),
1029 : ("Exit SpamBayes", self.OnExit),
}
message_map = {
win32con.WM_DESTROY: self.OnDestroy,
win32con.WM_COMMAND: self.OnCommand,
win32con.WM_USER+20 : self.OnTaskbarNotify,
}
# Register the Window class.
wc = WNDCLASS()
hinst = wc.hInstance = GetModuleHandle(None)
wc.lpszClassName = "SpambayesTaskbar"
wc.style = win32con.CS_VREDRAW | win32con.CS_HREDRAW;
wc.hCursor = LoadCursor( 0, win32con.IDC_ARROW )
wc.hbrBackground = win32con.COLOR_WINDOW
wc.lpfnWndProc = message_map
classAtom = RegisterClass(wc)
# Create the Window.
style = win32con.WS_OVERLAPPED | win32con.WS_SYSMENU
self.hwnd = CreateWindow(classAtom, "SpamBayes", style,
0, 0, win32con.CW_USEDEFAULT,
win32con.CW_USEDEFAULT, 0, 0, hinst, None)
UpdateWindow(self.hwnd)
# Try and find a custom icon
# XXX This needs to be done, but first someone needs to make a wee
# XXX spambayes icon
iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "pyc.ico" ))
if not os.path.isfile(iconPathName):
# Look in the source tree.
iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "..\\PC\\pyc.ico" ))
if os.path.isfile(iconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, 0, icon_flags)
else:
print "Can't find a spambayes icon file - using default"
hicon = LoadIcon(0, win32con.IDI_APPLICATION)
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
nid = (self.hwnd, 0, flags, win32con.WM_USER+20, hicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
# Start up pop3proxy
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
# XXX start that, and if not kick pop3proxy off in a separate thread.
#pop3proxy.prepare(state=pop3proxy.state)
#pop3proxy.start(pop3proxy.state)
#self.started = True
def OnDestroy(self, hwnd, msg, wparam, lparam):
nid = (self.hwnd, 0)
Shell_NotifyIcon(NIM_DELETE, nid)
PostQuitMessage(0)
def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
if lparam==win32con.WM_LBUTTONUP:
# We ignore left clicks
pass
elif lparam==win32con.WM_LBUTTONDBLCLK:
# Default behaviour is to open up the web interface
# XXX This should be set as the default (which then means bold
# XXX text) through the win32 calls, but win32all doesn't
# XXX include SetDefault(), which it needs to...
self.OpenInterface()
elif lparam==win32con.WM_RBUTTONUP:
menu = CreatePopupMenu()
ids = self.control_functions.keys()
ids.sort()
for id in ids:
(wording, function) = self.control_functions[id]
if function:
AppendMenu( menu, win32con.MF_STRING, id, wording)
else:
AppendMenu( menu, win32con.MF_SEPARATOR, id, wording)
pos = GetCursorPos()
SetForegroundWindow(self.hwnd)
TrackPopupMenu(menu, win32con.TPM_LEFTALIGN, pos[0], pos[1], 0,
self.hwnd, None)
PostMessage(self.hwnd, win32con.WM_NULL, 0, 0)
return 1
def OnCommand(self, hwnd, msg, wparam, lparam):
id = LOWORD(wparam)
try:
unused, function = self.control_functions[id]
except KeyError:
print "Unknown command -", id
function()
def OnExit(self):
DestroyWindow(self.hwnd)
sys.exit()
def StartStop(self):
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
# XXX start/stop that, and if not kick pop3proxy off in a separate
# XXX thread, or stop the thread that was started.
if self.started:
pop3proxy.stop(pop3proxy.state)
self.started = False
else:
pop3proxy.start(pop3proxy.state)
self.started = True
def OpenInterface(self):
webbrowser.open_new("http://localhost:%d/" % \
(options["html_ui", "port"],))
def OpenConfig(self):
webbrowser.open_new("http://localhost:%d/config" % \
(options["html_ui", "port"],))
def main():
w = MainWindow()
PumpMessages()
if __name__=='__main__':
main()
From anadelonbrin at users.sourceforge.net Mon Sep 1 04:33:50 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 06:33:53 2003
Subject: [Spambayes-checkins]
spambayes pop3proxy.py, 1.96, 1.97 smtpproxy.py, 1.13, 1.14
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv18001
Modified Files:
pop3proxy.py smtpproxy.py
Log Message:
Fix an (unreported) bug where restoring the defaults would not take effect until
restarting the proxy/filter.
Fix [ spambayes-Bugs-796996 ] smtp server not started until restart, which should
also fix Richie's bug about the smtpproxy not stopping when it was meant to, and
I think also Bruce Richardson's too many open connections problem.
Index: pop3proxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/pop3proxy.py,v
retrieving revision 1.96
retrieving revision 1.97
diff -C2 -d -r1.96 -r1.97
*** pop3proxy.py 31 Aug 2003 00:29:07 -0000 1.96
--- pop3proxy.py 1 Sep 2003 10:33:48 -0000 1.97
***************
*** 710,715 ****
global state
state = State()
- state.buildServerStrings()
- state.createWorkers()
# Close the existing listeners and create new ones. This won't
--- 710,713 ----
***************
*** 719,723 ****
--- 717,724 ----
proxy.close()
del proxyListeners[:]
+
+ prepare(state)
_createProxies(state.servers, state.proxyPorts)
+
return state
***************
*** 740,744 ****
import smtpproxy
servers, proxyPorts = smtpproxy.LoadServerInfo()
! smtpproxy.CreateProxies(servers, proxyPorts, state)
# setup info for the web interface
--- 741,746 ----
import smtpproxy
servers, proxyPorts = smtpproxy.LoadServerInfo()
! proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts,
! state))
# setup info for the web interface
Index: smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/smtpproxy.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** smtpproxy.py 1 Sep 2003 05:06:37 -0000 1.13
--- smtpproxy.py 1 Sep 2003 10:33:48 -0000 1.14
***************
*** 144,148 ****
from spambayes.Options import options
from pop3proxy import _addressPortStr, ServerLineReader
! from pop3proxy import _addressAndPort, proxyListeners
class SMTPProxyBase(Dibbler.BrighterAsyncChat):
--- 144,148 ----
from spambayes.Options import options
from pop3proxy import _addressPortStr, ServerLineReader
! from pop3proxy import _addressAndPort
class SMTPProxyBase(Dibbler.BrighterAsyncChat):
***************
*** 516,523 ****
--- 516,525 ----
if not isinstance(trainer, SMTPTrainer):
trainer = SMTPTrainer(trainer.bayes, trainer)
+ proxyListeners = []
for (server, serverPort), proxyPort in zip(servers, proxyPorts):
listener = BayesSMTPProxyListener(server, serverPort, proxyPort,
trainer)
proxyListeners.append(listener)
+ return proxyListeners
def main():
***************
*** 561,566 ****
servers, proxyPorts = LoadServerInfo()
trainer = SMTPTrainer(classifier)
! CreateProxies(servers, proxyPorts, trainer)
Dibbler.run()
if __name__ == '__main__':
--- 563,569 ----
servers, proxyPorts = LoadServerInfo()
trainer = SMTPTrainer(classifier)
! proxyListeners = CreateProxies(servers, proxyPorts, trainer)
Dibbler.run()
+
if __name__ == '__main__':
From anadelonbrin at users.sourceforge.net Mon Sep 1 04:33:51 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 06:33:57 2003
Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.22,
1.23
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv18001/spambayes
Modified Files:
UserInterface.py
Log Message:
Fix an (unreported) bug where restoring the defaults would not take effect until
restarting the proxy/filter.
Fix [ spambayes-Bugs-796996 ] smtp server not started until restart, which should
also fix Richie's bug about the smtpproxy not stopping when it was meant to, and
I think also Bruce Richardson's too many open connections problem.
Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** UserInterface.py 1 Sep 2003 09:05:51 -0000 1.22
--- UserInterface.py 1 Sep 2003 10:33:48 -0000 1.23
***************
*** 713,716 ****
--- 713,717 ----
self.writeOKHeaders('text/html')
self.write(html)
+ self.reReadOptions()
def verifyInput(self, parms, pmap):
From anadelonbrin at users.sourceforge.net Mon Sep 1 04:36:16 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 06:36:19 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.15,
1.16 WHAT_IS_NEW.txt, 1.12, 1.13
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv18376
Modified Files:
CHANGELOG.txt WHAT_IS_NEW.txt
Log Message:
Bring up to date, and add a clarifying statement about the dates.
Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** CHANGELOG.txt 1 Sep 2003 09:16:08 -0000 1.15
--- CHANGELOG.txt 1 Sep 2003 10:36:14 -0000 1.16
***************
*** 1,4 ****
--- 1,7 ----
+ [Note that all dates are in English, not American format - i.e. day/month/year]
+
Alpha Release 5
===============
+ Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart.
Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319.
Tony Meyer 01/09/2003 Added [ 796832 ] Word query should show all words starting with certain text
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** WHAT_IS_NEW.txt 1 Sep 2003 09:16:08 -0000 1.12
--- WHAT_IS_NEW.txt 1 Sep 2003 10:36:14 -0000 1.13
***************
*** 149,153 ****
790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316
A url containing the details of these bugs can be made by appending the
--- 149,153 ----
790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996
A url containing the details of these bugs can be made by appending the
From richiehindle at users.sourceforge.net Mon Sep 1 15:07:34 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Mon Sep 1 17:07:38 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py, 1.26,
1.27 classifier.py, 1.7, 1.8
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1240
Modified Files:
storage.py classifier.py
Log Message:
Bug #797890: After training on a message, ensure that the classifier's
state (nspam, nham) is written to the database. Otherwise, if training
goes wrong, you can get a wordinfo whose count is greater than
nspam/nham - for instance, when training on a 100-message mailbox
file, if the 99th message caused an exception you could get a clue
with a ham count of 99 but an nham of 0 (or whatever it was when
you started training).
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** storage.py 1 Sep 2003 09:39:31 -0000 1.26
--- storage.py 1 Sep 2003 21:07:32 -0000 1.27
***************
*** 214,221 ****
self.changed_words = {}
# Update the global state, then do the actual save.
self.db[self.statekey] = (classifier.PICKLE_VERSION,
self.nspam, self.nham)
- self.db.sync()
def _wordinfoget(self, word):
if isinstance(word, unicode):
--- 214,230 ----
self.changed_words = {}
# Update the global state, then do the actual save.
+ self._write_state_key()
+ self.db.sync()
+
+ def _write_state_key(self):
self.db[self.statekey] = (classifier.PICKLE_VERSION,
self.nspam, self.nham)
+ def _post_training(self):
+ """This is called after training on a wordstream. We ensure that the
+ database is in a consistent state at this point by writing the state
+ key."""
+ self._write_state_key()
+
def _wordinfoget(self, word):
if isinstance(word, unicode):
Index: classifier.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** classifier.py 1 Sep 2003 09:05:51 -0000 1.7
--- classifier.py 1 Sep 2003 21:07:32 -0000 1.8
***************
*** 411,414 ****
--- 411,415 ----
self._wordinfoset(word, record)
+ self._post_training()
def _remove_msg(self, wordstream, is_spam):
***************
*** 437,440 ****
--- 438,449 ----
self._wordinfoset(word, record)
+ self._post_training()
+
+ def _post_training(self):
+ """This is called after training on a wordstream. Subclasses might
+ want to ensure that their databases are in a consistent state at
+ this point. Introduced to fix bug #797890."""
+ pass
+
def _getclues(self, wordstream):
mindist = options.minimum_prob_strength
From richiehindle at users.sourceforge.net Mon Sep 1 15:31:46 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Mon Sep 1 17:31:53 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.14,1.15
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6441
Modified Files:
Version.py
Log Message:
Lots of fixes and improvements to the POP3 proxy and the web interface
since May - got to be worth an uprev.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** Version.py 9 Aug 2003 03:26:54 -0000 1.14
--- Version.py 1 Sep 2003 21:31:44 -0000 1.15
***************
*** 44,52 ****
},
"POP3 Proxy" : {
! "Version": 0.1,
! "Description": "SpamBayes POP3 Proxy Beta1",
! "Date": "May 2003",
! "InterfaceVersion": 0.02,
! "InterfaceDescription": "SpamBayes POP3 Proxy Web Interface Alpha2",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
--- 44,52 ----
},
"POP3 Proxy" : {
! "Version": 0.2,
! "Description": "SpamBayes POP3 Proxy Beta2",
! "Date": "September 2003",
! "InterfaceVersion": 0.03,
! "InterfaceDescription": "SpamBayes POP3 Proxy Web Interface Alpha3",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
From richiehindle at users.sourceforge.net Mon Sep 1 15:36:45 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Mon Sep 1 17:36:48 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt, 1.16,
1.17 WHAT_IS_NEW.txt, 1.13, 1.14
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv7277
Modified Files:
CHANGELOG.txt WHAT_IS_NEW.txt
Log Message:
Update for 797890 fix.
Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** CHANGELOG.txt 1 Sep 2003 10:36:14 -0000 1.16
--- CHANGELOG.txt 1 Sep 2003 21:36:42 -0000 1.17
***************
*** 3,6 ****
--- 3,7 ----
Alpha Release 5
===============
+ Richie Hindle 01/09/2003 Fix [ 797890 ] "assert hamcount <= nham" problem.
Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart.
Richie Hindle 01/09/2003 Integrated [ 791393 ] HTTP Authentication, which closes #791319.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** WHAT_IS_NEW.txt 1 Sep 2003 10:36:14 -0000 1.13
--- WHAT_IS_NEW.txt 1 Sep 2003 21:36:43 -0000 1.14
***************
*** 41,44 ****
--- 41,45 ----
POP3 Proxy / SMTP Proxy / POP3 Proxy Service
--------------------------------------------
+ o Fixed "assert hamcount <= nham" problem.
o Starting and stopping the POP3 Proxy service (for Windows NT, Windows
2000 and Windows XP users) has been improved. Most noticeably, this
From xenogeist at users.sourceforge.net Mon Sep 1 15:43:25 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Mon Sep 1 17:43:39 2003
Subject: [Spambayes-checkins] spambayes/windows/resources - New directory
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv8327/resources
Log Message:
Directory /cvsroot/spambayes/spambayes/windows/resources added to the repository
From xenogeist at users.sourceforge.net Mon Sep 1 15:46:28 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Mon Sep 1 17:46:32 2003
Subject: [Spambayes-checkins] spambayes/windows/resources dialogs.h, NONE,
1.1 dialogs.rc, NONE, 1.1 sbicon.ico, NONE, 1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv8836/windows/resources
Added Files:
dialogs.h dialogs.rc sbicon.ico
Log Message:
Fixed Tony's tray program to work around Python 2.3. Added code to run the proxy in a seperate thread (I'll leave interacting with a service for another day). Includes the icon voted most likely change.
--- NEW FILE: dialogs.h ---
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by dialogs.rc
//
#define IDD_MANAGER 101
#define IDI_SBICON 102
#define IDC_PROGRESS 1000
#define IDC_PROGRESS_TEXT 1001
#define IDC_STATIC_HAM 1002
#define IDC_STATIC_SPAM 1003
#define IDC_BROWSE_HAM 1004
#define IDC_BROWSE_SPAM 1005
#define IDC_START 1006
#define IDC_BUT_REBUILD 1007
#define IDC_BUT_RESCORE 1008
#define IDC_VERSION 1009
#define IDC_BUT_TRAIN_FROM_SPAM_FOLDER 1010
#define IDC_BUT_TRAIN_TO_SPAM_FOLDER 1011
#define IDC_BUT_TRAIN_NOW 1012
#define IDC_BUT_FILTER_ENABLE 1013
#define IDC_FILTER_STATUS 1014
#define IDC_BUT_FILTER_NOW 1015
#define IDC_BUT_FILTER_DEFINE 1016
#define IDC_BUT_ABOUT 1017
#define IDC_BUT_ACT_SCORE 1018
#define IDC_BUT_ACT_ALL 1019
#define IDC_BUT_UNREAD 1020
#define IDC_BUT_UNSEEN 1021
#define IDC_BUT_FILTERNOW 1022
#define IDC_SLIDER_CERTAIN 1023
#define IDC_EDIT_CERTAIN 1024
#define IDC_ACTION_CERTAIN 1025
#define IDC_TOFOLDER_CERTAIN 1026
#define IDC_FOLDER_CERTAIN 1027
#define IDC_BROWSE_CERTAIN 1028
#define IDC_SLIDER_UNSURE 1029
#define IDC_EDIT_UNSURE 1030
#define IDC_ACTION_UNSURE 1031
#define IDC_TOFOLDER_UNSURE 1032
#define IDC_FOLDER_UNSURE 1033
#define IDC_BROWSE_UNSURE 1034
#define IDC_TRAINING_STATUS 1035
#define IDC_FOLDER_NAMES 1036
#define IDC_BROWSE 1037
#define IDC_FOLDER_WATCH 1038
#define IDC_BROWSE_WATCH 1039
#define IDC_LIST_FOLDERS 1040
#define IDC_BUT_SEARCHSUB 1041
#define IDC_BUT_CLEARALL 1042
#define IDC_STATUS1 1043
#define IDC_STATUS2 1044
#define IDC_BUT_CLEARALL2 1045
#define IDC_BUT_NEW 1046
#define IDC_MARK_SPAM_AS_READ 1047
#define IDC_SAVE_SPAM_SCORE 1048
#define IDC_MARK_UNSURE_AS_READ 1051
#define IDC_ADVANCED_BTN 1055
#define IDC_DELAY1_SLIDER 1056
#define IDC_DELAY1_TEXT 1057
#define IDC_DELAY2_SLIDER 1058
#define IDC_DELAY2_TEXT 1059
#define IDC_INBOX_TIMER_ONLY 1060
#define IDC_VERBOSE_LOG 1061
#define IDB_SBLOGO 1062
#define IDC_LOGO_GRAPHIC 1063
#define IDC_USE_DELAY1 1064
#define IDC_USE_DELAY2 1065
#define IDC_TAB 1068
#define IDC_BACK_BTN 1069
#define IDC_BUT_WIZARD 1070
#define IDC_SHOW_DATA_FOLDER 1071
#define IDC_ABOUT_BTN 1072
#define IDC_BUT_RESET 1073
#define IDC_DEL_SPAM_RS 1074
#define IDC_RECOVER_RS 1075
#define IDC_HIDDEN 1076
#define IDC_FORWARD_BTN 1077
#define IDC_PAGE_PLACEHOLDER 1078
#define IDC_SHOW_DATA_FOLDER2 1079
#define IDC_BUT_SHOW_DIAGNOSTICS 1080
#define IDC_BUT_PREPARATION 1081
#define IDC_FOLDER_HAM 1083
#define IDC_BUT_UNTRAINED 1088
#define IDC_BUT_TRAIN 1089
#define IDC_BUT_TIMER_ENABLED 1091
#define IDC_WIZ_GRAPHIC 1092
#define IDC_BUT_VIEW_LOG 1093
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 103
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1094
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
--- NEW FILE: dialogs.rc ---
// Microsoft Visual C++ generated resource script.
//
#include "dialogs.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "winres.h"
// spambayes dialog definitions
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
/////////////////////////////////////////////////////////////////////////////
//
// Dialog
//
IDD_MANAGER DIALOGEX 0, 0, 275, 260
STYLE DS_SETFONT | DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
EXSTYLE WS_EX_CONTEXTHELP
CAPTION "SpamBayes"
FONT 8, "Tahoma", 400, 0, 0x0
BEGIN
DEFPUSHBUTTON "Close",IDOK,216,239,50,14
PUSHBUTTON "Cancel",IDCANCEL,155,239,50,14,NOT WS_VISIBLE
CONTROL "",IDC_TAB,"SysTabControl32",0x0,8,7,258,228
PUSHBUTTON "About",IDC_ABOUT_BTN,9,239,50,14
END
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"dialogs.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""winres.h""\r\n"
"// spambayes dialog definitions\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Icon
//
// Icon with lowest ID value placed first to ensure application icon
// remains consistent on all systems.
IDI_SBICON ICON "sbicon.ico"
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED
--- NEW FILE: sbicon.ico ---
(This appears to be a binary file; contents omitted.)
From xenogeist at users.sourceforge.net Mon Sep 1 15:46:28 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Mon Sep 1 17:46:33 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv8836/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Fixed Tony's tray program to work around Python 2.3. Added code to run the proxy in a seperate thread (I'll leave interacting with a service for another day). Includes the icon voted most likely change.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** pop3proxy_tray.py 1 Sep 2003 10:04:58 -0000 1.1
--- pop3proxy_tray.py 1 Sep 2003 21:46:26 -0000 1.2
***************
*** 9,13 ****
# Foundation license.
! __author__ = "Tony Meyer "
__credits__ = "Mark Hammond, all the Spambayes folk."
--- 9,13 ----
# Foundation license.
! __author__ = "Tony Meyer , Adam Walker"
__credits__ = "Mark Hammond, all the Spambayes folk."
***************
*** 24,27 ****
--- 24,28 ----
import sys
import webbrowser
+ import thread
import win32con
***************
*** 37,40 ****
--- 38,43 ----
from spambayes.Options import options
+ WM_TASKBAR_NOTIFY = win32con.WM_USER + 20
+
class MainWindow(object):
def __init__(self):
***************
*** 53,80 ****
win32con.WM_DESTROY: self.OnDestroy,
win32con.WM_COMMAND: self.OnCommand,
! win32con.WM_USER+20 : self.OnTaskbarNotify,
}
- # Register the Window class.
- wc = WNDCLASS()
- hinst = wc.hInstance = GetModuleHandle(None)
- wc.lpszClassName = "SpambayesTaskbar"
- wc.style = win32con.CS_VREDRAW | win32con.CS_HREDRAW;
- wc.hCursor = LoadCursor( 0, win32con.IDC_ARROW )
- wc.hbrBackground = win32con.COLOR_WINDOW
- wc.lpfnWndProc = message_map
- classAtom = RegisterClass(wc)
-
# Create the Window.
! style = win32con.WS_OVERLAPPED | win32con.WS_SYSMENU
! self.hwnd = CreateWindow(classAtom, "SpamBayes", style,
! 0, 0, win32con.CW_USEDEFAULT,
! win32con.CW_USEDEFAULT, 0, 0, hinst, None)
! UpdateWindow(self.hwnd)
# Try and find a custom icon
# XXX This needs to be done, but first someone needs to make a wee
# XXX spambayes icon
! iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "pyc.ico" ))
if not os.path.isfile(iconPathName):
# Look in the source tree.
--- 56,74 ----
win32con.WM_DESTROY: self.OnDestroy,
win32con.WM_COMMAND: self.OnCommand,
! WM_TASKBAR_NOTIFY : self.OnTaskbarNotify,
}
# Create the Window.
! hinst = GetModuleHandle(None)
! # this will replaced with a real configure dialog later
! # this is mainly to work around not being able to register a window class
! # with python 2.3
! dialogTemplate = [['SpamBayes', (14, 10, 246, 187), -1865809852 & ~win32con.WS_VISIBLE, None, (8, 'Tahoma')],]
! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0, message_map)
# Try and find a custom icon
# XXX This needs to be done, but first someone needs to make a wee
# XXX spambayes icon
! iconPathName = os.path.abspath( "resources\\sbicon.ico" )
if not os.path.isfile(iconPathName):
# Look in the source tree.
***************
*** 88,93 ****
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, win32con.WM_USER+20, hicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
# Start up pop3proxy
--- 82,88 ----
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
+ self.started = False
# Start up pop3proxy
***************
*** 95,106 ****
# XXX This should determine if we are using the service, and if so
# XXX start that, and if not kick pop3proxy off in a separate thread.
! #pop3proxy.prepare(state=pop3proxy.state)
! #pop3proxy.start(pop3proxy.state)
! #self.started = True
def OnDestroy(self, hwnd, msg, wparam, lparam):
! nid = (self.hwnd, 0)
! Shell_NotifyIcon(NIM_DELETE, nid)
! PostQuitMessage(0)
def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
--- 90,100 ----
# XXX This should determine if we are using the service, and if so
# XXX start that, and if not kick pop3proxy off in a separate thread.
! pop3proxy.prepare(state=pop3proxy.state)
! self.StartProxyThread()
def OnDestroy(self, hwnd, msg, wparam, lparam):
! nid = (self.hwnd, 0)
! Shell_NotifyIcon(NIM_DELETE, nid)
! PostQuitMessage(0)
def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
***************
*** 142,145 ****
--- 136,144 ----
DestroyWindow(self.hwnd)
sys.exit()
+
+ def StartProxyThread(self):
+ args = (pop3proxy.state,)
+ thread.start_new_thread(pop3proxy.start, args)
+ self.started = True
def StartStop(self):
***************
*** 152,157 ****
self.started = False
else:
! pop3proxy.start(pop3proxy.state)
! self.started = True
def OpenInterface(self):
--- 151,155 ----
self.started = False
else:
! self.StartProxyThread()
def OpenInterface(self):
From xenogeist at users.sourceforge.net Mon Sep 1 15:56:22 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Mon Sep 1 17:56:33 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv10626/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Be nice and stop the proxy before exiting the app.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** pop3proxy_tray.py 1 Sep 2003 21:46:26 -0000 1.2
--- pop3proxy_tray.py 1 Sep 2003 21:56:20 -0000 1.3
***************
*** 134,137 ****
--- 134,140 ----
def OnExit(self):
+ if self.started:
+ pop3proxy.stop(pop3proxy.state)
+ self.started = False
DestroyWindow(self.hwnd)
sys.exit()
From anadelonbrin at users.sourceforge.net Mon Sep 1 17:58:24 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 19:58:28 2003
Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.14,1.15
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30418
Modified Files:
WHAT_IS_NEW.txt
Log Message:
I'd forgotten about the version info :) smtpproxy has definately gone up a version,
and the same for the imapfilter interface, although not the filter itself.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** WHAT_IS_NEW.txt 1 Sep 2003 21:36:43 -0000 1.14
--- WHAT_IS_NEW.txt 1 Sep 2003 23:58:22 -0000 1.15
***************
*** 150,154 ****
790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996
A url containing the details of these bugs can be made by appending the
--- 150,155 ----
790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996,
! 797890
A url containing the details of these bugs can be made by appending the
From anadelonbrin at users.sourceforge.net Mon Sep 1 17:58:25 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 19:58:30 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.15,1.16
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30418/spambayes
Modified Files:
Version.py
Log Message:
I'd forgotten about the version info :) smtpproxy has definately gone up a version,
and the same for the imapfilter interface, although not the filter itself.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** Version.py 1 Sep 2003 21:31:44 -0000 1.15
--- Version.py 1 Sep 2003 23:58:22 -0000 1.16
***************
*** 59,65 ****
},
"SMTP Proxy" : {
! "Version": 0.01,
! "Description": "SpamBayes SMTP Proxy Alpha1",
! "Date": "May 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
},
--- 59,65 ----
},
"SMTP Proxy" : {
! "Version": 0.02,
! "Description": "SpamBayes SMTP Proxy Alpha2",
! "Date": "September 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
},
***************
*** 68,73 ****
"Description": "SpamBayes IMAP Filter Alpha1",
"Date": "May 2003",
! "InterfaceVersion": 0.01,
! "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha1",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
--- 68,73 ----
"Description": "SpamBayes IMAP Filter Alpha1",
"Date": "May 2003",
! "InterfaceVersion": 0.02,
! "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
From anadelonbrin at users.sourceforge.net Mon Sep 1 18:15:14 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 20:15:18 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv383/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Remove assumptions about which directory we are running in when finding the
icon, and get rid of the code to deal with not finding it.
Update the menu to reflect the current start/stop status.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** pop3proxy_tray.py 1 Sep 2003 21:56:20 -0000 1.3
--- pop3proxy_tray.py 2 Sep 2003 00:15:12 -0000 1.4
***************
*** 40,43 ****
--- 40,45 ----
WM_TASKBAR_NOTIFY = win32con.WM_USER + 20
+ START_STOP_ID = 1024
+
class MainWindow(object):
def __init__(self):
***************
*** 46,50 ****
# that the order is controlled by the id. Any items were the
# function is None will appear as separators.
! self.control_functions = {1024 : ("Start SpamBayes", self.StartStop),
1025 : ("-", None),
1026 : ("View information ...", self.OpenInterface),
--- 48,52 ----
# that the order is controlled by the id. Any items were the
# function is None will appear as separators.
! self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
1025 : ("-", None),
1026 : ("View information ...", self.OpenInterface),
***************
*** 61,83 ****
# Create the Window.
hinst = GetModuleHandle(None)
! # this will replaced with a real configure dialog later
! # this is mainly to work around not being able to register a window class
! # with python 2.3
! dialogTemplate = [['SpamBayes', (14, 10, 246, 187), -1865809852 & ~win32con.WS_VISIBLE, None, (8, 'Tahoma')],]
! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0, message_map)
! # Try and find a custom icon
! # XXX This needs to be done, but first someone needs to make a wee
! # XXX spambayes icon
! iconPathName = os.path.abspath( "resources\\sbicon.ico" )
! if not os.path.isfile(iconPathName):
! # Look in the source tree.
! iconPathName = os.path.abspath(os.path.join( os.path.split(sys.executable)[0], "..\\PC\\pyc.ico" ))
if os.path.isfile(iconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0, 0, icon_flags)
! else:
! print "Can't find a spambayes icon file - using default"
! hicon = LoadIcon(0, win32con.IDI_APPLICATION)
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
--- 63,85 ----
# Create the Window.
hinst = GetModuleHandle(None)
! # This will replaced with a real configure dialog later
! # This is mainly to work around not being able to register a window
! # class with Python 2.3
! dialogTemplate = [['SpamBayes', (14, 10, 246, 187),
! -1865809852 & ~win32con.WS_VISIBLE, None,
! (8, 'Tahoma')],]
! self.hwnd = CreateDialogIndirect(hinst, dialogTemplate, 0,
! message_map)
! # Get the custom icon
! iconPathName = "%s\\windows\\resources\\sbicon.ico" % \
! (os.path.dirname(pop3proxy.__file__),)
! # When 1.0a6 is released, the above line will need to change to:
! ## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
! ## (os.path.dirname(pop3proxy.__file__),)
if os.path.isfile(iconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
***************
*** 153,158 ****
--- 155,164 ----
pop3proxy.stop(pop3proxy.state)
self.started = False
+ self.control_functions[START_STOP_ID] = ("Start SpamBayes",
+ self.StartStop)
else:
self.StartProxyThread()
+ self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
+ self.StartStop)
def OpenInterface(self):
From anadelonbrin at users.sourceforge.net Mon Sep 1 19:03:19 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 21:03:22 2003
Subject: [Spambayes-checkins] spambayes imapfilter.py, 1.51,
1.52 smtpproxy.py, 1.14, 1.15
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv7577
Modified Files:
imapfilter.py smtpproxy.py
Log Message:
Clarify a print statement in smtpproxy.
imapfilter: correct options names. correct some comments. change some error
checking to be more certain that we will raise the error and not imaplib if something
goes wrong. address [ 788845 ] imapfilter training fails, hopefully fixing it.
Index: imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v
retrieving revision 1.51
retrieving revision 1.52
diff -C2 -d -r1.51 -r1.52
*** imapfilter.py 13 Aug 2003 06:36:46 -0000 1.51
--- imapfilter.py 2 Sep 2003 01:03:16 -0000 1.52
***************
*** 60,66 ****
much nicer.
o IMAP over SSL is untested.
! o Develop a test script, like testtools/pop3proxytest.py that runs
! through some tests (perhaps with a *real* imap server, rather than
! a dummy one). This would make it easier to carry out the tests
against each server whenever a change is made.
o IMAP supports authentication via other methods than the plain-text
--- 60,66 ----
much nicer.
o IMAP over SSL is untested.
! o Develop a test script, like spambayes/test/test_pop3proxy.py that
! runs through some tests (perhaps with a *real* imap server, rather
! than a dummy one). This would make it easier to carry out the tests
against each server whenever a change is made.
o IMAP supports authentication via other methods than the plain-text
***************
*** 311,315 ****
if self.got_substance:
return
! if self.uid is None or self.id is None:
print "Cannot get substance of message without an id and an UID"
return
--- 311,315 ----
if self.got_substance:
return
! if not self.uid or not self.id:
print "Cannot get substance of message without an id and an UID"
return
***************
*** 318,321 ****
--- 318,322 ----
# the status of the message. Unfortunately, it appears that not
# all IMAP servers support this, even though it is in RFC1730
+ # Actually, it's not: we should be using BODY.PEEK
try:
response = imap.uid("FETCH", self.uid, self.rfc822_command)
***************
*** 340,345 ****
self.epilogue = new_msg.epilogue
self._default_type = new_msg._default_type
! if not self.has_key(options["pop3proxy", "mailid_header_name"]):
! self[options["pop3proxy", "mailid_header_name"]] = self.id
self.got_substance = True
if options["globals", "verbose"]:
--- 341,346 ----
self.epilogue = new_msg.epilogue
self._default_type = new_msg._default_type
! if not self.has_key(options["Headers", "mailid_header_name"]):
! self[options["Headers", "mailid_header_name"]] = self.id
self.got_substance = True
if options["globals", "verbose"]:
***************
*** 360,364 ****
raise RuntimeError, """Can't save a message that doesn't
have a folder."""
! if self.id is None:
raise RuntimeError, """Can't save a message that doesn't have
an id."""
--- 361,365 ----
raise RuntimeError, """Can't save a message that doesn't
have a folder."""
! if not self.id:
raise RuntimeError, """Can't save a message that doesn't have
an id."""
***************
*** 403,407 ****
imap.SelectFolder(self.folder.name)
response = imap.uid("SEARCH", "(UNDELETED HEADER " + \
! options["pop3proxy", "mailid_header_name"] + \
" " + self.id + ")")
self._check(response, 'search')
--- 404,408 ----
imap.SelectFolder(self.folder.name)
response = imap.uid("SEARCH", "(UNDELETED HEADER " + \
! options["Headers", "mailid_header_name"] + \
" " + self.id + ")")
self._check(response, 'search')
***************
*** 416,419 ****
--- 417,430 ----
ids = new_id.split(' ')
new_id = ids[-1]
+ # Ok, now we're in trouble if we still haven't found it.
+ # We make a huge assumption that the new message is the one
+ # with the highest UID (they are sequential, so this will be
+ # ok as long as another message hasn't also arrived)
+ if new_id == "":
+ response = imap.uid("SEARCH", "ALL")
+ new_id = response[1][0]
+ if new_id.find(' ') > -1:
+ ids = new_id.split(' ')
+ new_id = ids[-1]
self.uid = new_id
***************
*** 481,485 ****
msg.setFolder(self)
msg.uid = key
! r = re.compile(re.escape(options["pop3proxy",
"mailid_header_name"]) + \
"\:\s*(\d+(\-\d)?)")
--- 492,496 ----
msg.setFolder(self)
msg.uid = key
! r = re.compile(re.escape(options["Headers",
"mailid_header_name"]) + \
"\:\s*(\d+(\-\d)?)")
***************
*** 648,653 ****
sys.exit()
! bdbname = options["pop3proxy", "persistent_storage_file"]
! useDBM = options["pop3proxy", "persistent_use_database"]
doTrain = False
doClassify = False
--- 659,664 ----
sys.exit()
! bdbname = options["Storage", "persistent_storage_file"]
! useDBM = options["Storage", "persistent_use_database"]
doTrain = False
doClassify = False
Index: smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/smtpproxy.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** smtpproxy.py 1 Sep 2003 10:33:48 -0000 1.14
--- smtpproxy.py 2 Sep 2003 01:03:17 -0000 1.15
***************
*** 453,457 ****
print "Could not find message (%s); perhaps it was " + \
"deleted from the POP3Proxy cache or the IMAP " + \
! "server." % (id, )
def train_message_in_pop3proxy_cache(self, id, isSpam):
--- 453,457 ----
print "Could not find message (%s); perhaps it was " + \
"deleted from the POP3Proxy cache or the IMAP " + \
! "server. This means that no training was done." % (id, )
def train_message_in_pop3proxy_cache(self, id, isSpam):
From anadelonbrin at users.sourceforge.net Mon Sep 1 19:31:06 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 21:31:15 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.16,1.17
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12061/spambayes
Modified Files:
Version.py
Log Message:
Use the storage.open_storage function in imapfilter, which means it joins the
ranks of giving a nice error when someone tries to use dumbdbm, and users
can use the new SQL classifiers.
Fix Adam Many's problem with listing folder names (the server was replying with a
literal, which we didn't expect).
Stop being stingy and give imapfilter a revup, otherwise people will think it hasn't
changed at all, when there have been very minor changes.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** Version.py 1 Sep 2003 23:58:22 -0000 1.16
--- Version.py 2 Sep 2003 01:31:04 -0000 1.17
***************
*** 65,71 ****
},
"IMAP Filter" : {
! "Version": 0.01,
! "Description": "SpamBayes IMAP Filter Alpha1",
! "Date": "May 2003",
"InterfaceVersion": 0.02,
"InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2",
--- 65,71 ----
},
"IMAP Filter" : {
! "Version": 0.02,
! "Description": "SpamBayes IMAP Filter Alpha2",
! "Date": "September 2003",
"InterfaceVersion": 0.02,
"InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2",
From anadelonbrin at users.sourceforge.net Mon Sep 1 19:31:06 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 21:31:17 2003
Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt, 1.15,
1.16 imapfilter.py, 1.52, 1.53
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12061
Modified Files:
WHAT_IS_NEW.txt imapfilter.py
Log Message:
Use the storage.open_storage function in imapfilter, which means it joins the
ranks of giving a nice error when someone tries to use dumbdbm, and users
can use the new SQL classifiers.
Fix Adam Many's problem with listing folder names (the server was replying with a
literal, which we didn't expect).
Stop being stingy and give imapfilter a revup, otherwise people will think it hasn't
changed at all, when there have been very minor changes.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** WHAT_IS_NEW.txt 1 Sep 2003 23:58:22 -0000 1.15
--- WHAT_IS_NEW.txt 2 Sep 2003 01:31:03 -0000 1.16
***************
*** 151,155 ****
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996,
! 797890
A url containing the details of these bugs can be made by appending the
--- 151,155 ----
779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996,
! 797890, 788845
A url containing the details of these bugs can be made by appending the
Index: imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v
retrieving revision 1.52
retrieving revision 1.53
diff -C2 -d -r1.52 -r1.53
*** imapfilter.py 2 Sep 2003 01:03:16 -0000 1.52
--- imapfilter.py 2 Sep 2003 01:31:03 -0000 1.53
***************
*** 246,251 ****
--- 246,263 ----
folders = []
for fol in all_folders:
+ # Sigh. Some servers may give us back the folder name as a
+ # literal, so we need to crunch this out.
+ if isinstance(fol, ()):
+ r = re.compile(r"{\d+}")
+ m = r.search(fol[0])
+ if not m:
+ # Something is wrong here! Skip this folder
+ continue
+ fol = '%s"%s"' % (fol[0][:m.start()], fol[1])
r = re.compile(r"\(([\w\\ ]*)\) ")
m = r.search(fol)
+ if not m:
+ # Something is not good with this folder, so skip it.
+ continue
name_attributes = fol[:m.end()-1]
# IMAP is a truly odd protocol. The delimiter is
***************
*** 720,728 ****
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
!
! if useDBM:
! classifier = storage.DBDictClassifier(bdbname)
! else:
! classifier = storage.PickledClassifier(bdbname)
if options["globals", "verbose"]:
--- 732,737 ----
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
!
! classifier = storage.open_storage(bdbname, useDBM)
if options["globals", "verbose"]:
From anadelonbrin at users.sourceforge.net Mon Sep 1 21:42:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 1 23:42:05 2003
Subject: [Spambayes-checkins] website download.ht,1.15,1.16
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv32236
Modified Files:
download.ht
Log Message:
Fix [ 796434 ] email package link is defunct
(Thanks Barry).
Index: download.ht
===================================================================
RCS file: /cvsroot/spambayes/website/download.ht,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** download.ht 12 Aug 2003 09:57:33 -0000 1.15
--- download.ht 2 Sep 2003 03:41:55 -0000 1.16
***************
*** 12,16 ****
Either:
- Python 2.2.2, Python 2.3b1, or a CVS build of python, or
!
- Python 2.2, 2.2.1, plus the latest email package.
Once you've downloaded and unpacked the source archive, do the regular setup.py build; setup.py install dance, then:
--- 12,16 ----
Either:
- Python 2.2.2, Python 2.3b1, or a CVS build of python, or
!
- Python 2.2, 2.2.1, plus the latest email package.
Once you've downloaded and unpacked the source archive, do the regular setup.py build; setup.py install dance, then:
From anadelonbrin at users.sourceforge.net Tue Sep 2 00:48:58 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 2 02:49:02 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,NONE,1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv24581/windows
Added Files:
autoconfigure.py
Log Message:
This script will read the configuration file for a mail client and changes it to use
the spambayes pop3/smtp proxies (or imapfilter at some point), and sets up
the bayescustomize.ini file to the correct servers/ports. Where possible, it
also creates filtering rules as well.
Currently works for Eudora 5.2 (including the filtering rule), Mozilla 1.3 Mail,
and Opera 7.11 Mail (M2). Outlook Express is the next (much more difficult)
target.
--- NEW FILE: autoconfigure.py ---
#!/usr/bin/env python
"""Automatically set up the user's mail client and SpamBayes.
Currently works with:
o Eudora (POP3/SMTP only)
o Mozilla Mail (POP3/SMTP only)
o Opera Mail (M2) (POP3/SMTP only)
To do:
o Establish which mail client(s) are to be setup.
o Locate the appropriate configuration directory
(e.g. for Eudora this is probably either the application directory,
or c:\documents and settings\username\application data\qualcomm\eudora,
i.e. sh_appdata\qualcomm\eudora)
o This will create some unnecessary proxies in some cases. For example,
if I have my client set up to get mail from pop.example.com for the
user 'tmeyer' and the user 'tonym', two proxies will be created, but
only one is necessary. We should check the existing proxies before
adding a new one.
o Figure out Outlook Express's pop3uidl.dbx file and how to hook into it
(use the oe_mailbox.py module)
o Other mail clients? Other platforms?
o This won't work all that well if multiple mail clients are used (they
will end up trying to use the same ports). In such a case, we really
need to keep track of if the server is being proxied already, and
reuse ports, but this is complicated.
o We currently don't make any moves to protect the original file, so if
something does wrong, it's corrupted. We also write into the file,
rather than a temporary one and then copy across. This should all be
fixed.
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "All the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
## Tested with:
## o Eudora 5.2 on Windows XP
## o Mozilla 1.3 on Windows XP
## o Opera 7.11 on Windows XP
import re
import os
import types
import socket
import StringIO
import ConfigParser
from spambayes.Options import options, optionsPathname
def move_to_next_free_port(port):
# Increment port until we get to one that isn't taken
# I doubt this will work if there is a firewall that prevents
# localhost connecting to particular ports, but I'm not sure
# how else we can do this - Richie says that bind() doesn't
# necessarily fail if the port is already bound.
while True:
try:
port += 1
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", port))
s.close()
except socket.error:
return port
# Let's be safe and use high ports, starting at 1110 and 1025, and going up
# as required.
pop_proxy_port = move_to_next_free_port(1109)
smtp_proxy_port = move_to_next_free_port(1024)
def configure_eudora(config_location):
"""Configure Eudora to use the SpamBayes POP3 and SMTP proxies, and
configure SpamBayes to proxy the servers that Eudora was connecting to.
"""
ini_filename = "%s%seudora.ini" % (config_location, os.sep)
c = ConfigParser.ConfigParser()
c.read(ini_filename)
translate = {("PopServer", "POPPort") : "pop3proxy",
("SMTPServer", "SMTPPort") : "smtpproxy",
}
pop_proxy = pop_proxy_port
smtp_proxy = smtp_proxy_port
for sect in c.sections():
if sect.startswith("Persona-") or sect == "Settings":
if c.get(sect, "UsesIMAP") == "0":
# Eudora stores the POP3 server name in two places.
# Why? Who knows? We do the popaccount one
# separately, because it also has the username.
p = c.get(sect, "popaccount")
c.set(sect, "popaccount", "%s@localhost" % \
(p[:p.index('@')],))
for ((eud_name, eud_port), us_name in translate.items():
try:
port = c.get(sect, eud_port)
except ConfigParser.NoOptionError:
port = None
if us_name.lower()[:4] == "pop3":
if port is None:
port = 110
pop_proxy = move_to_next_free_port(pop_proxy)
proxy_port = pop_proxy
else:
if port is None:
port = 25
smtp_proxy = move_to_next_free_port(smtp_proxy)
proxy_port = smtp_proxy
server = "%s:%s" % (c.get(sect, eud_name), port)
options[us_name, "remote_servers"] += (server,)
options[us_name, "listen_ports"] += (proxy_port,)
if options["globals", "verbose"]:
print "[%s] Proxy %s on localhost:%s" % \
(sect, server, proxy_port)
c.set(sect, eud_name, "localhost")
c.set(sect, eud_port, proxy_port)
else:
# Setup imapfilter instead
pass
out = file(ini_filename, "w")
c.write(out)
out.close()
options.update_file(optionsPathname)
# Setup filtering rule
# This assumes that the spam and unsure folders already exist!
# (Creating them shouldn't be that difficult - it's just a mbox file,
# and I think the .toc file is automatically created). Left for
# another day, however.
filter_filename = "%s%sFilters.pce" % (config_location, os.sep)
spam_folder_name = "Junk Mail"
unsure_folder_name = "Possible Junk"
header_name = options["Headers", "classification_header_name"]
spam_tag = options["Headers", "header_spam_string"]
unsure_tag = options["Headers", "header_unsure_string"]
# We are assuming that a rules file already exists, otherwise there
# is a bit more to go at the top.
filter_rules = "rule SpamBayes-Spam\n" \
"transfer %s.mbx\n" \
"incoming\n" \
"header %s\n" \
"verb contains\n" \
"value %s\n" \
"conjunction ignore\n" \
"header \n" \
"verb contains\n" \
"value \n" \
"rule SpamBayes-Unsure\n" \
"transfer %s.mbx\n" \
"incoming\n" \
"header %s\n" \
"verb contains\n" \
"value %s\n" \
"conjunction ignore\n" \
"header \n" \
"verb contains\n" \
"value \n" % (spam_folder_name, header_name, spam_tag,
unsure_folder_name, header_name, unsure_tag)
filter_file = file(filter_filename, "a")
filter_file.write(filter_rules)
filter_file.close()
def configure_mozilla(config_location):
"""Configure Mozilla to use the SpamBayes POP3 and SMTP proxies, and
configure SpamBayes to proxy the servers that Mozilla was connecting
to."""
prefs_file = file("%s%sprefs.js" % (config_location, os.sep), "r")
prefs = prefs_file.read()
prefs_file.close()
save_prefs = prefs
pop_accounts = {}
smtp_accounts = {}
r = re.compile(r"user_pref\(\"mail.server.server(\d+).(real)?hostname\", \"([^\"]*)\"\);")
current_pos = 0
while True:
m = r.search(prefs[current_pos:])
if not m:
break
server_num = m.group(1)
real = m.group(2) or ''
server = m.group(3)
current_pos += m.end()
old_pref = 'user_pref("mail.server.server%s.%shostname", "%s");' % \
(server_num, real, server)
# Find the port, if there is one
port_string = 'user_pref("mail.server.server%s.port", ' % \
(server_num,)
port_loc = prefs.find(port_string)
if port_loc == -1:
port = "110"
old_port = None
else:
loc_plus_len = port_loc + len(port_string)
end_of_number = loc_plus_len + prefs[loc_plus_len:].index(')')
port = prefs[loc_plus_len : end_of_number]
old_port = "%s%s);" % (port_string, port)
# Find the type of connection
type_string = 'user_pref("mail.server.server%s.type", "' % \
(server_num,)
type_loc = prefs.find(type_string)
if type_loc == -1:
# no type, so ignore this one
continue
type_loc += len(type_string)
account_type = prefs[type_loc : \
type_loc + prefs[type_loc:].index('"')]
if account_type == "pop3":
new_pref = 'user_pref("mail.server.server%s.%shostname", ' \
'"127.0.0.1");' % (server_num, real)
if not pop_accounts.has_key(server_num) or real:
pop_accounts[server_num] = (new_pref, old_pref,
old_port, server, port)
elif account_type == "imap":
# Setup imapfilter instead
pass
proxy_port = pop_proxy_port
for num, (pref, old_pref, old_port, server, port) in pop_accounts.items():
server = "%s:%s" % (server, port)
proxy_port = move_to_next_free_port(proxy_port)
port_pref = 'user_pref("mail.server.server%s.port", %s);' % \
(num, proxy_port)
options["pop3proxy", "remote_servers"] += (server,)
options["pop3proxy", "listen_ports"] += (proxy_port,)
if old_port is None:
pref = "%s\n%s" % (pref, port_pref)
else:
save_prefs = save_prefs.replace(old_port, port_pref)
save_prefs = save_prefs.replace(old_pref, pref)
if options["globals", "verbose"]:
print "[%s] Proxy %s on localhost:%s" % \
(num, server, proxy_port)
# Do the SMTP server.
# Mozilla recommends that only advanced users setup more than one,
# so we'll just set that one up. Advanced users can setup SpamBayes
# themselves .
prefs = save_prefs
r = re.compile(r"user_pref\(\"mail.smtpserver.smtp(\d+).hostname\", \"([^\"]*)\"\);")
current_pos = 0
while True:
m = r.search(prefs[current_pos:])
if not m:
break
current_pos = m.end()
server_num = m.group(1)
server = m.group(2)
old_pref = 'user_pref("mail.smtpserver.smtp%s.hostname", ' \
'"%s");' % (server_num, server)
new_pref = 'user_pref("mail.smtpserver.smtp%s.hostname", ' \
'"127.0.0.1");' % (server_num,)
# Find the port
port_string = 'user_pref("mail.smtpserver.smtp1.port", '
port_loc = prefs.find(port_string)
if port_loc == -1:
port = "25"
old_port = None
else:
loc_plus_len = port_loc + len(port_string)
end_of_number = loc_plus_len + prefs[loc_plus_len:].index(')')
port = prefs[loc_plus_len : end_of_number]
old_port = 'user_pref("mail.smtpserver.smtp%s.port", %s);' % \
(server_num, port)
smtp_accounts[server_num] = (new_pref, old_pref, old_port,
server, port)
proxy_port = smtp_proxy_port
for num, (pref, old_pref, old_port, server, port) in smtp_accounts.items():
server = "%s:%s" % (server, port)
proxy_port = move_to_next_free_port(proxy_port)
port_pref = 'user_pref("mail.smtpserver.smtp%s.port", %s);' % \
(num, proxy_port)
options["smtpproxy", "remote_servers"] += (server,)
options["smtpproxy", "listen_ports"] += (proxy_port,)
if old_port is None:
pref = "%s\n%s" % (pref, port_pref)
else:
save_prefs = save_prefs.replace(old_port, port_pref)
save_prefs = save_prefs.replace(old_pref, pref)
if options["globals", "verbose"]:
print "[%s] Proxy %s on localhost:%s" % \
(num, server, proxy_port)
prefs_file = file("%s%sprefs.js" % (config_location, os.sep), "w")
prefs_file.write(save_prefs)
prefs_file.close()
options.update_file(optionsPathname)
# Setup filtering rules.
# Assumes that the folders already exist! I don't know how difficult
# it would be to create new Mozilla mail folders.
filter_filename = "%s%smsgFilterRules.dat" % (config_location, os.sep)
store_name = "" # how do we get this?
spam_folder_url = "mailbox:////%s//Junk%20Mail" % (store_name,)
unsure_folder_url = "mailbox:////%s//Possible%20Junk" % (store_name,)
header_name = options["Headers", "classification_header_name"]
spam_tag = options["Headers", "header_spam_string"]
unsure_tag = options["Headers", "header_unsure_string"]
rule = 'name="SpamBayes-Spam"\n' \
'enabled="yes"\n' \
'type="1"\n' \
'action="Move to folder"\n' \
'actionValue="%s"\n' \
'condition="OR (\"%s\",contains,%s)"\n' \
'name="SpamBayes-Unsure"\n' \
'enabled="yes"\n' \
'type="1"\n' \
'action="Move to folder"\n' \
'actionValue="%s"\n' \
'condition="OR (\"%s\",contains,%s)"\n' % \
(spam_folder_url, header_name, spam_tag,
unsure_folder_url, header_name, unsure_tag)
# This should now be written to the file, but I'm not sure how we
# determine which subdirectory it goes into - does it have to go
# into them all?
# We are assuming that a rules file already exists, otherwise there
# is a bit more to go at the top.
def configure_m2(config_location):
"""Configure M2 (Opera's mailer) to use the SpamBayes POP3 and SMTP
proxies, and configure SpamBayes to proxy the servers that M2 was
connecting to."""
ini_filename = os.path.join(config_location, "Mail", "accounts.ini")
ini_file = file(ini_filename, "r")
faked_up = StringIO.StringIO()
faked_up.write(";") # Missing at the start
faked_up.write(ini_file.read())
faked_up.seek(0)
ini_file.close()
c = ConfigParser.ConfigParser()
c.readfp(faked_up)
translate = {("Incoming Servername", "Incoming Port") : "pop3proxy",
("Outgoing Servername", "Outgoing Port") : "smtpproxy",
}
pop_proxy = pop_proxy_port
smtp_proxy = smtp_proxy_port
for sect in c.sections():
if sect.startswith("Account") and sect != "Accounts":
if c.get(sect, "Incoming Protocol") == "POP":
for (m2_name, m2_port), us_name in translate.items():
try:
port = c.get(sect, m2_port)
except ConfigParser.NoOptionError:
port = None
if us_name.lower()[:4] == "pop3":
if port is None:
port = 110
pop_proxy = move_to_next_free_port(pop_proxy)
proxy_port = pop_proxy
else:
if port is None:
port = 25
smtp_proxy = move_to_next_free_port(smtp_proxy)
proxy_port = smtp_proxy
server = "%s:%s" % (c.get(sect, m2_name), port)
options[us_name, "remote_servers"] += (server,)
options[us_name, "listen_ports"] += (proxy_port,)
if options["globals", "verbose"]:
print "[%s] Proxy %s on localhost:%s" % \
(sect, server, proxy_port)
c.set(sect, m2_name, "localhost")
c.set(sect, m2_port, proxy_port)
elif c.get(sect, "Incoming Protocol") == "IMAP":
# Setup imapfilter instead
pass
out = file(ini_filename, "w")
c.write(out)
out.close()
options.update_file(optionsPathname)
# Setting up a filter in M2 is very simple, but I'm not sure what the
# right rule is - M2 doesn't move mail, it just displays a subset.
# If someone can describe the best all-purpose rule, I'll pop it in
# here.
if __name__ == "__main__":
#configure_eudora(eudora_ini_dir)
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
pass
From anadelonbrin at users.sourceforge.net Tue Sep 2 00:52:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 2 02:52:59 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv25160/windows
Modified Files:
autoconfigure.py
Log Message:
Opps. Syntax error from a last minute edit.
Index: autoconfigure.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** autoconfigure.py 2 Sep 2003 06:48:55 -0000 1.1
--- autoconfigure.py 2 Sep 2003 06:52:53 -0000 1.2
***************
*** 61,65 ****
def move_to_next_free_port(port):
! # Increment port until we get to one that isn't taken
# I doubt this will work if there is a firewall that prevents
# localhost connecting to particular ports, but I'm not sure
--- 61,65 ----
def move_to_next_free_port(port):
! # Increment port until we get to one that isn't taken.
# I doubt this will work if there is a firewall that prevents
# localhost connecting to particular ports, but I'm not sure
***************
*** 104,108 ****
c.set(sect, "popaccount", "%s@localhost" % \
(p[:p.index('@')],))
! for ((eud_name, eud_port), us_name in translate.items():
try:
port = c.get(sect, eud_port)
--- 104,108 ----
c.set(sect, "popaccount", "%s@localhost" % \
(p[:p.index('@')],))
! for (eud_name, eud_port), us_name in translate.items():
try:
port = c.get(sect, eud_port)
***************
*** 398,401 ****
--- 398,404 ----
# here.
+ def configure_outlook_express(config_location):
+ dbx_filename = os.path.join(config_location, "pop3uidl.dbx")
+
if __name__ == "__main__":
***************
*** 403,405 ****
--- 406,409 ----
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
+ #configure_outlook_express(oe_ini_dir)
pass
From anadelonbrin at users.sourceforge.net Tue Sep 2 02:29:03 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 2 04:29:10 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv7322/windows
Modified Files:
autoconfigure.py
Log Message:
We can now auto-setup Outlook Express as well :)
Index: autoconfigure.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** autoconfigure.py 2 Sep 2003 06:52:53 -0000 1.2
--- autoconfigure.py 2 Sep 2003 08:29:01 -0000 1.3
***************
*** 7,10 ****
--- 7,11 ----
o Mozilla Mail (POP3/SMTP only)
o Opera Mail (M2) (POP3/SMTP only)
+ o Outlook Express (POP3/SMTP only)
To do:
***************
*** 47,53 ****
## Tested with:
! ## o Eudora 5.2 on Windows XP
! ## o Mozilla 1.3 on Windows XP
! ## o Opera 7.11 on Windows XP
import re
--- 48,55 ----
## Tested with:
! ## o Eudora 5.2 on Windows XP
! ## o Mozilla 1.3 on Windows XP
! ## o Opera 7.11 on Windows XP
! ## o Outlook Express 6 on Windows XP
import re
***************
*** 398,409 ****
# here.
! def configure_outlook_express(config_location):
! dbx_filename = os.path.join(config_location, "pop3uidl.dbx")
if __name__ == "__main__":
#configure_eudora(eudora_ini_dir)
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
! #configure_outlook_express(oe_ini_dir)
pass
--- 400,481 ----
# here.
! def configure_outlook_express(key):
! """Configure OE to use the SpamBayes POP3 and SMTP proxies, and
! configure SpamBayes to proxy the servers that OE was connecting to."""
! # OE stores its configuration in the registry, not a file.
!
! key = key + "\\Software\\Microsoft\\Internet Account Manager\\Accounts"
!
! import win32api
! import win32con
!
! translate = {("POP3 Server", "POP3 Port") : "pop3proxy",
! ("SMTP Server", "SMTP Port") : "smtpproxy",
! }
!
! pop_proxy = pop_proxy_port
! smtp_proxy = smtp_proxy_port
!
! reg = win32api.RegOpenKeyEx(win32con.HKEY_USERS, key)
! account_index = 0
! while True:
! # Loop through all the accounts
! config = {}
! try:
! subkey_name = "%s\\%s" % \
! (key, win32api.RegEnumKey(reg, account_index))
! except win32api.error:
! break
! account_index += 1
! index = 0
! subkey = win32api.RegOpenKeyEx(win32con.HKEY_USERS, subkey_name, 0,
! win32con.KEY_READ | win32con.KEY_SET_VALUE)
! while True:
! # Loop through all the keys
! try:
! raw = win32api.RegEnumValue(subkey, index)
! except win32api.error:
! break
! config[raw[0]] = (raw[1], raw[2])
! index += 1
!
! # Process this account
! if config.has_key("POP3 Server"):
! for (server_key, port_key), sect in translate.items():
! server = "%s:%s" % (config[server_key][0],
! config[port_key][0])
! if sect[:4] == "pop3":
! pop_proxy = move_to_next_free_port(pop_proxy)
! proxy = pop_proxy
! else:
! smtp_proxy = move_to_next_free_port(smtp_proxy)
! proxy = smtp_proxy
! options[sect, "remote_servers"] += (server,)
! options[sect, "listen_ports"] += (proxy,)
! win32api.RegSetValueEx(subkey, server_key, 0,
! win32con.REG_SZ, "127.0.0.1")
! win32api.RegSetValueEx(subkey, port_key, 0,
! win32con.REG_SZ, str(proxy))
! if options["globals", "verbose"]:
! print "[%s] Proxy %s on localhost:%s" % \
! (config["Account Name"][0], server, proxy)
! elif config.has_key("IMAP Server"):
! # Setup imapfilter instead.
! pass
!
! options.update_file(optionsPathname)
!
! # Outlook Express rules are done in much the same way. Should one
! # be set up to work with notate_to or notate_subject? (and set that
! # option, obviously)
if __name__ == "__main__":
+ # XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224"
+ # XXX but I presume it's different for everyone? I'll have to check on
+ # XXX another machine.
#configure_eudora(eudora_ini_dir)
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
! configure_outlook_express()
pass
From xenogeist at users.sourceforge.net Tue Sep 2 14:09:31 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Tue Sep 2 16:09:36 2003
Subject: [Spambayes-checkins] spambayes/windows/resources .cvsignore, NONE,
1.1 sb-started.ico, NONE, 1.1 sb-stopped.ico, NONE, 1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv30326/windows/resources
Added Files:
.cvsignore sb-started.ico sb-stopped.ico
Log Message:
icons by Romain GUY.
--- NEW FILE: .cvsignore ---
dialogs.aps
--- NEW FILE: sb-started.ico ---
(This appears to be a binary file; contents omitted.)
--- NEW FILE: sb-stopped.ico ---
(This appears to be a binary file; contents omitted.)
From xenogeist at users.sourceforge.net Tue Sep 2 14:12:15 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Tue Sep 2 16:12:18 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv31083/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Switch icons based on if the proxy is running or not. Provide some info found on the information page in the tooltip of the icon.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** pop3proxy_tray.py 2 Sep 2003 00:15:12 -0000 1.4
--- pop3proxy_tray.py 2 Sep 2003 20:12:13 -0000 1.5
***************
*** 73,90 ****
# Get the custom icon
! iconPathName = "%s\\windows\\resources\\sbicon.ico" % \
(os.path.dirname(pop3proxy.__file__),)
# When 1.0a6 is released, the above line will need to change to:
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
## (os.path.dirname(pop3proxy.__file__),)
! if os.path.isfile(iconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
! hicon = LoadImage(hinst, iconPathName, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
self.started = False
# Start up pop3proxy
--- 73,95 ----
# Get the custom icon
! startedIconPathName = "%s\\windows\\resources\\sb-started.ico" % \
! (os.path.dirname(pop3proxy.__file__),)
! stoppedIconPathName = "%s\\windows\\resources\\sb-stopped.ico" % \
(os.path.dirname(pop3proxy.__file__),)
# When 1.0a6 is released, the above line will need to change to:
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
## (os.path.dirname(pop3proxy.__file__),)
! if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
! self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
! self.hstoppedicon = LoadImage(hinst, stoppedIconPathName, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
self.started = False
+ self.tip = None
# Start up pop3proxy
***************
*** 93,97 ****
# XXX start that, and if not kick pop3proxy off in a separate thread.
pop3proxy.prepare(state=pop3proxy.state)
! self.StartProxyThread()
def OnDestroy(self, hwnd, msg, wparam, lparam):
--- 98,124 ----
# XXX start that, and if not kick pop3proxy off in a separate thread.
pop3proxy.prepare(state=pop3proxy.state)
! self.StartStop()
!
! def BuildToolTip(self):
! tip = None
! if self.started == True:
! #%i spam %i unsure %i session %i active
! tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
! (pop3proxy.state.numSpams, pop3proxy.state.numHams, pop3proxy.state.numUnsure,
! pop3proxy.state.totalSessions, pop3proxy.state.activeSessions)
! else:
! tip = "SpamBayes is not running"
! return tip
!
!
! def UpdateIcon(self, hicon=None):
! flags = NIF_TIP
! if hicon is not None:
! flags |= NIF_ICON
! else:
! hicon = 0
! self.tip = self.BuildToolTip()
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
! Shell_NotifyIcon(NIM_MODIFY, nid)
def OnDestroy(self, hwnd, msg, wparam, lparam):
***************
*** 101,104 ****
--- 128,134 ----
def OnTaskbarNotify(self, hwnd, msg, wparam, lparam):
+ if lparam==win32con.WM_MOUSEMOVE:
+ if self.tip != self.BuildToolTip():
+ self.UpdateIcon()
if lparam==win32con.WM_LBUTTONUP:
# We ignore left clicks
***************
*** 157,164 ****
--- 187,196 ----
self.control_functions[START_STOP_ID] = ("Start SpamBayes",
self.StartStop)
+ self.UpdateIcon(self.hstoppedicon)
else:
self.StartProxyThread()
self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
self.StartStop)
+ self.UpdateIcon(self.hstartedicon)
def OpenInterface(self):
From richiehindle at users.sourceforge.net Tue Sep 2 16:47:50 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Tue Sep 2 18:47:54 2003
Subject: [Spambayes-checkins] spambayes/spambayes Dibbler.py,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv31385
Modified Files:
Dibbler.py
Log Message:
Remove gratuitous quotes from "You must log in."""
Index: Dibbler.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Dibbler.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** Dibbler.py 1 Sep 2003 06:07:27 -0000 1.6
--- Dibbler.py 2 Sep 2003 22:47:48 -0000 1.7
***************
*** 333,337 ****
def getCancelMessage(self):
"""Override: Specify the cancel message for an HTTP Authentication."""
! return "You must log in."""
--- 333,337 ----
def getCancelMessage(self):
"""Override: Specify the cancel message for an HTTP Authentication."""
! return "You must log in."
From anadelonbrin at users.sourceforge.net Tue Sep 2 18:37:02 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 2 20:37:05 2003
Subject: [Spambayes-checkins] spambayes imapfilter.py,1.53,1.54
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv15432
Modified Files:
imapfilter.py
Log Message:
We would crash if we hadn't set anything up and didn't prompt for password.
(I could swear that I've fixed this bug before...)
Index: imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/imapfilter.py,v
retrieving revision 1.53
retrieving revision 1.54
diff -C2 -d -r1.53 -r1.54
*** imapfilter.py 2 Sep 2003 01:31:03 -0000 1.53
--- imapfilter.py 3 Sep 2003 00:36:59 -0000 1.54
***************
*** 753,756 ****
--- 753,757 ----
pwd = pwd[0]
else:
+ pwd = None
if not launchUI:
print "You need to specify both a server and a username."
From kennypitt at hotmail.com Wed Sep 3 12:00:19 2003
From: kennypitt at hotmail.com (Kenny Pitt)
Date: Thu Sep 4 14:13:50 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py, NONE,
1.1
In-Reply-To:
Message-ID: <000001c3722c$1a46c940$300a10ac@spidynamics.com>
Tony Meyer wrote:
> Update of /cvsroot/spambayes/spambayes/windows
> In directory sc8-pr-cvs1:/tmp/cvs-serv13593/windows
>
> Added Files:
> pop3proxy_tray.py
> Log Message:
> A basic shell for a pop3proxy tray application, which can open up the
> configuration page, and start and stop the proxy. The start/stop
> code needs a lot more work, as part of the pop3proxy binary process.
>
> Note that this will cause Python 2.3 to unexpectedly terminate, but
> will work fine in Python 2.2. See [ 798452 ] win32gui_taskbar.py
> broken in Python 2.3
>
>
> --- NEW FILE: pop3proxy_tray.py ---
[snip]
Anyone know if something is up with the SourceForge CVS servers? I
thought the time delay for us read-only types was supposed to be getting
better, but I received this checkin email on Monday at 6:19am and here
it is Wednesday morning and "cvs update" still hasn't pulled the first
version of the file.
--
Kenny Pitt
From mhammond at users.sourceforge.net Thu Sep 4 06:14:14 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 4 17:15:26 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py, 1.106,
1.107 filter.py, 1.31, 1.32 manager.py, 1.83, 1.84 msgstore.py,
1.71, 1.72 tester.py, 1.16, 1.17
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv21153
Modified Files:
addin.py filter.py manager.py msgstore.py tester.py
Log Message:
Start using msgstore exception objects:
* msgstore no longer returns None for GetMessage/GetFolder - now always
returns an object, or raises a msgstore exception. Any message
operation could yield such an error (as the object may be moved after
the message object has been instantiated)
* addin now catches these exceptions rather than checking None
* filter now catches these exceptions rather than pythoncom.com_error
exceptions directly.
* Support in the test suite for "forcing" exceptions. When the test suite
is running, we can simulate MAPI errors at various times. Thus allows
us to get far more code coverage (and IIRC, Skip has a tool to actually
record code coverage - I must check that out!)
* test suite *only* filters our test messages, leaving all other messages
as "unprocessed". When the test suite ends, these are processed as
"missed messages".
* test suite hacks an event object into the filter process. This speeds
up the test suite enormously (an order of magnitude at least)
* manager gives a nod to the test suite.
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.106
retrieving revision 1.107
diff -C2 -d -r1.106 -r1.107
*** addin.py 1 Sep 2003 03:54:12 -0000 1.106
--- addin.py 4 Sep 2003 12:14:11 -0000 1.107
***************
*** 166,208 ****
manager.LogDebug(2, "ProcessMessage starting for message '%s'" \
% msgstore_message.subject)
! if not msgstore_message.IsFilterCandidate():
! manager.LogDebug(1, "Skipping message '%s' - we don't filter ones like that!" \
! % msgstore_message.subject)
! return
!
! if HaveSeenMessage(msgstore_message, manager):
! # Already seen this message - user probably moving it back
! # after incorrect classification.
! # If enabled, re-train as Ham
! # otherwise just ignore.
! if manager.config.training.train_recovered_spam:
! import train
! if train.been_trained_as_spam(msgstore_message, manager.classifier_data):
! need_train = True
! else:
! prop = msgstore_message.GetField(manager.config.general.field_score_name)
! # We may not have been able to save the score - re-score now
! if prop is None:
! prop = manager.score(msgstore_message)
! # If it was not previously classified as either 'Spam' or
! # 'Unsure', then this event is unlikely to be the user
! # re-classifying (and in fact it may simply be the Outlook
! # rules moving the item.
! need_train = manager.config.filter.unsure_threshold < prop * 100
!
! if need_train:
! TrainAsHam(msgstore_message, manager)
! else:
! subject = msgstore_message.subject
! manager.LogDebug(1, "Message '%s' was previously seen, but " \
! "did not need to be trained as ham" % subject)
! return
! if manager.config.filter.enabled:
! import filter
! disposition = filter.filter_message(msgstore_message, manager)
! print "Message '%s' had a Spam classification of '%s'" \
! % (msgstore_message.GetSubject(), disposition)
! else:
! print "Spam filtering is disabled - ignoring new message"
manager.LogDebug(2, "ProcessMessage finished for", msgstore_message)
--- 166,211 ----
manager.LogDebug(2, "ProcessMessage starting for message '%s'" \
% msgstore_message.subject)
! try:
! if not msgstore_message.IsFilterCandidate():
! manager.LogDebug(1, "Skipping message '%s' - we don't filter ones like that!" \
! % msgstore_message.subject)
! return
!
! if HaveSeenMessage(msgstore_message, manager):
! # Already seen this message - user probably moving it back
! # after incorrect classification.
! # If enabled, re-train as Ham
! # otherwise just ignore.
! if manager.config.training.train_recovered_spam:
! import train
! if train.been_trained_as_spam(msgstore_message, manager.classifier_data):
! need_train = True
! else:
! prop = msgstore_message.GetField(manager.config.general.field_score_name)
! # We may not have been able to save the score - re-score now
! if prop is None:
! prop = manager.score(msgstore_message)
! # If it was not previously classified as either 'Spam' or
! # 'Unsure', then this event is unlikely to be the user
! # re-classifying (and in fact it may simply be the Outlook
! # rules moving the item.
! need_train = manager.config.filter.unsure_threshold < prop * 100
!
! if need_train:
! TrainAsHam(msgstore_message, manager)
! else:
! subject = msgstore_message.subject
! manager.LogDebug(1, "Message '%s' was previously seen, but " \
! "did not need to be trained as ham" % subject)
! return
! if manager.config.filter.enabled:
! import filter
! disposition = filter.filter_message(msgstore_message, manager)
! print "Message '%s' had a Spam classification of '%s'" \
! % (msgstore_message.GetSubject(), disposition)
! else:
! print "Spam filtering is disabled - ignoring new message"
! except manager.message_store.NotFoundException:
! manager.LogDebug(1, "ProcessMessage had the message moved out from underneath us")
manager.LogDebug(2, "ProcessMessage finished for", msgstore_message)
***************
*** 340,345 ****
while 1:
item = self.timer_generator.next()
! if not HaveSeenMessage(item, self.manager):
! break
except StopIteration:
# No items left in our generator
--- 343,352 ----
while 1:
item = self.timer_generator.next()
! try:
! if not HaveSeenMessage(item, self.manager):
! break
! except self.manager.message_store.NotFoundException:
! # ignore messages move underneath us
! self.manager.LogDebug(1, "The new message is skipping a message that moved underneath us")
except StopIteration:
# No items left in our generator
***************
*** 365,371 ****
# generator would miss it, so we process it synchronously.
if not self.use_timer or not item.UnRead:
! msgstore_message = self.manager.message_store.GetMessage(item)
! if msgstore_message is not None:
ProcessMessage(msgstore_message, self.manager)
else:
self._StartTimer()
--- 372,383 ----
# generator would miss it, so we process it synchronously.
if not self.use_timer or not item.UnRead:
! ms = self.manager.message_store
! try:
! msgstore_message = ms.GetMessage(item)
ProcessMessage(msgstore_message, self.manager)
+ except ms.MsgStoreException, details:
+ print "Unexpected error fetching message"
+ traceback.print_exc()
+ print details
else:
self._StartTimer()
***************
*** 382,385 ****
--- 394,400 ----
if not self.manager.config.training.train_manual_spam:
return
+ # XXX - Theoretically we could get "not found" exception here,
+ # but we have never guarded for it, and never seen it. If it does
+ # happen life will go on, so for now we continue to ignore it.
msgstore_message = self.manager.message_store.GetMessage(item)
if not msgstore_message.IsFilterCandidate():
***************
*** 539,543 ****
print "Executing automated tests..."
tester.test(manager)
- print "Tests worked."
except:
traceback.print_exc()
--- 554,557 ----
***************
*** 572,579 ****
# Delete this item as spam.
spam_folder = None
spam_folder_id = self.manager.config.filter.spam_folder_id
if spam_folder_id:
! spam_folder = msgstore.GetFolder(spam_folder_id)
! if not spam_folder:
self.manager.ReportError("You must configure the Spam folder",
"Invalid Configuration")
--- 586,598 ----
# Delete this item as spam.
spam_folder = None
+ # It is unlikely that the spam folder is not specified, as the UI
+ # will prevent enabling. But it could be invalid.
spam_folder_id = self.manager.config.filter.spam_folder_id
if spam_folder_id:
! try:
! spam_folder = msgstore.GetFolder(spam_folder_id)
! except msgstore.MsgStoreException:
! pass
! if spam_folder is None:
self.manager.ReportError("You must configure the Spam folder",
"Invalid Configuration")
***************
*** 631,657 ****
# that the source folder == dest folder - restore to
# the inbox in this case.
- subject = msgstore_message.GetSubject()
- restore_folder = msgstore_message.GetRememberedFolder()
- if restore_folder is None or \
- msgstore_message.GetFolder() == restore_folder:
- print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
- restore_folder = inbox_folder
-
- # Must train before moving, else we lose the message!
- print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
- TrainAsHam(msgstore_message, self.manager)
- # Do the new message state if necessary.
try:
! if new_msg_state == "Read":
! msgstore_message.SetReadState(True)
! elif new_msg_state == "Unread":
! msgstore_message.SetReadState(False)
! else:
! if new_msg_state not in ["", "None", None]:
! print "*** Bad new_msg_state value: %r" % (new_msg_state,)
! except pythoncom.com_error:
! print "*** Failed to set the message state to '%s' for message '%s'" % (new_msg_state, subject)
! # Now move it.
! msgstore_message.MoveToReportingError(self.manager, restore_folder)
# Note the move will possibly also trigger a re-train
# but we are smart enough to know we have already done it.
--- 650,681 ----
# that the source folder == dest folder - restore to
# the inbox in this case.
try:
! subject = msgstore_message.GetSubject()
! restore_folder = msgstore_message.GetRememberedFolder()
! if restore_folder is None or \
! msgstore_message.GetFolder() == restore_folder:
! print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
! restore_folder = inbox_folder
!
! # Must train before moving, else we lose the message!
! print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
! TrainAsHam(msgstore_message, self.manager)
! # Do the new message state if necessary.
! try:
! if new_msg_state == "Read":
! msgstore_message.SetReadState(True)
! elif new_msg_state == "Unread":
! msgstore_message.SetReadState(False)
! else:
! if new_msg_state not in ["", "None", None]:
! print "*** Bad new_msg_state value: %r" % (new_msg_state,)
! except msgstore.MsgStoreException, details:
! print "*** Failed to set the message state to '%s' for message '%s'" % (new_msg_state, subject)
! print details
! # Now move it.
! msgstore_message.MoveToReportingError(self.manager, restore_folder)
! except msgstore.NotFoundException:
! # Message moved under us - ignore.
! self.manager.LogDebug(1, "Recover from spam had message moved from underneath us - ignored")
# Note the move will possibly also trigger a re-train
# but we are smart enough to know we have already done it.
***************
*** 930,938 ****
ret = []
for i in range(sel.Count):
item = sel.Item(i+1)
! msgstore_message = self.manager.message_store.GetMessage(item)
! if msgstore_message and msgstore_message.IsFilterCandidate():
! ret.append(msgstore_message)
if len(ret) == 0:
--- 954,970 ----
ret = []
+ ms = self.manager.message_store
for i in range(sel.Count):
item = sel.Item(i+1)
! try:
! msgstore_message = ms.GetMessage(item)
! if msgstore_message.IsFilterCandidate():
! ret.append(msgstore_message)
! except ms.NotFoundException:
! pass
! except ms.MsgStoreException, details:
! print "Unexpected error fetching message"
! traceback.print_exc()
! print details
if len(ret) == 0:
Index: filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/filter.py,v
retrieving revision 1.31
retrieving revision 1.32
diff -C2 -d -r1.31 -r1.32
*** filter.py 25 Aug 2003 01:33:11 -0000 1.31
--- filter.py 4 Sep 2003 12:14:12 -0000 1.32
***************
*** 10,15 ****
True, False = 1, 0
- import pythoncom # for the exceptions.
-
def filter_message(msg, mgr, all_actions=True):
config = mgr.config.filter
--- 10,13 ----
***************
*** 29,32 ****
--- 27,31 ----
attr_prefix = None
+ ms = mgr.message_store
try:
try:
***************
*** 43,59 ****
msg.RememberMessageCurrentFolder()
msg.Save()
! except pythoncom.com_error, (hr, exc_msg, exc, arg_err):
! # This seems to happen for IMAP mails (0x800cccd3)
! # and also for hotmail messages (0x8004dff7)
! known_failure_codes = -2146644781, -2147164169
! # I also heard a rumour hotmail works if we do 2 saves
! if hr not in known_failure_codes:
! print "Unexpected MAPI error saving the spam score for", msg
! print hr, exc_msg, exc
! else:
! # So we can see if it still happens :)
! mgr.LogDebug(1, "Note: known (but still not understood) " \
! "error 0x%x saving the spam score." % hr)
! # No need for a traceback in this case.
# Clear dirty flag anyway
msg.dirty = False
--- 42,54 ----
msg.RememberMessageCurrentFolder()
msg.Save()
! except ms.ReadOnlyException:
! # read-only message - not much we can do!
! # Clear dirty flag anyway
! mgr.LogDebug(1, "Message is read-only - could not save Spam score")
! msg.dirty = False
! except ms.MsgStoreException, details:
! # Some other error saving - this is nasty.
! print "Unexpected MAPI error saving the spam score for", msg
! print details
# Clear dirty flag anyway
msg.dirty = False
***************
*** 68,73 ****
pass
elif action.startswith("co"): # copied
! dest_folder = mgr.message_store.GetFolder(folder_id)
! if dest_folder is None:
print "ERROR: Unable to open the folder to Copy the " \
"message - this message was not copied"
--- 63,69 ----
pass
elif action.startswith("co"): # copied
! try:
! dest_folder = ms.GetFolder(folder_id)
! except ms.MsgStoreException:
print "ERROR: Unable to open the folder to Copy the " \
"message - this message was not copied"
***************
*** 75,80 ****
msg.CopyToReportingError(mgr, dest_folder)
elif action.startswith("mo"): # Moved
! dest_folder = mgr.message_store.GetFolder(folder_id)
! if dest_folder is None:
print "ERROR: Unable to open the folder to Move the " \
"message - this message was not moved"
--- 71,77 ----
msg.CopyToReportingError(mgr, dest_folder)
elif action.startswith("mo"): # Moved
! try:
! dest_folder = ms.GetFolder(folder_id)
! except ms.MsgStoreException:
print "ERROR: Unable to open the folder to Move the " \
"message - this message was not moved"
Index: manager.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v
retrieving revision 1.83
retrieving revision 1.84
diff -C2 -d -r1.83 -r1.84
*** manager.py 29 Aug 2003 00:10:33 -0000 1.83
--- manager.py 4 Sep 2003 12:14:12 -0000 1.84
***************
*** 329,332 ****
--- 329,333 ----
self.outlook = outlook
self.dialog_parser = None
+ self.test_suite_running = False
import_early_core_spambayes_stuff()
***************
*** 411,416 ****
--- 412,425 ----
def ReportError(self, message, title = None):
+ if self.test_suite_running:
+ print "ReportError:", repr(message)
+ print "(but test suite running - not reported)"
+ return
ReportError(message, title)
def ReportInformation(self, message, title=None):
+ if self.test_suite_running:
+ print "ReportInformation:", repr(message)
+ print "(but test suite running - not reported)"
+ return
ReportInformation(message, title)
def AskQuestion(self, message, title=None):
***************
*** 443,446 ****
--- 452,459 ----
if key is None: key = msg
# Always print the message and traceback.
+ if self.test_suite_running:
+ print "ReportErrorOnce:", repr(msg)
+ print "(but test suite running - not reported)"
+ return
print "ERROR:", repr(msg)
traceback.print_exc()
***************
*** 505,513 ****
names = []
for eid in folder_ids:
! folder = self.message_store.GetFolder(eid)
! if folder is None:
! name = ""
! else:
name = folder.name
names.append(name)
ret = '; '.join(names)
--- 518,526 ----
names = []
for eid in folder_ids:
! try:
! folder = self.message_store.GetFolder(eid)
name = folder.name
+ except self.message_store.MsgStoreException:
+ name = ""
names.append(name)
ret = '; '.join(names)
***************
*** 530,535 ****
# (which really is OK!)
assert self.outlook is not None, "I need outlook :("
! msgstore_folder = self.message_store.GetFolder(folder_id)
! if msgstore_folder is None:
print "Checking a folder for our field failed - "\
"there is no such folder."
--- 543,549 ----
# (which really is OK!)
assert self.outlook is not None, "I need outlook :("
! try:
! msgstore_folder = self.message_store.GetFolder(folder_id)
! except self.message_store.MsgStoreException:
print "Checking a folder for our field failed - "\
"there is no such folder."
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.71
retrieving revision 1.72
diff -C2 -d -r1.71 -r1.72
*** msgstore.py 1 Sep 2003 07:29:48 -0000 1.71
--- msgstore.py 4 Sep 2003 12:14:12 -0000 1.72
***************
*** 10,94 ****
True, False = 1, 0
! # Nod to our automated test suite - we *do* want these messages filtered!
! test_suite_running = False
!
! # Abstract definition - can be moved out when we have more than one sub-class
! # External interface to this module is almost exclusively via a "folder ID"
!
! class MsgStoreException(Exception):
! pass
!
! class NotFoundException(MsgStoreException):
! pass
!
! class MsgStore:
! # Stash exceptions in the class for ease of use by consumers.
! MsgStoreException = MsgStoreException
! NotFoundException = NotFoundException
! def __init__(self):
! pass
! def Close(self):
! # Close this object and free everything
! raise NotImplementedError
! def GetFolderGenerator(self, folder_ids, include_sub):
! # Return a generator of MsgStoreFolder objects.
! raise NotImplementedError
! def GetFolder(self, folder_id):
! # Return a single folder given the ID.
! raise NotImplementedError
! def GetMessage(self, message_id):
! # Return a single message given the ID.
! raise NotImplementedError
!
! class MsgStoreFolder:
! def __init__(self):
! self.name = ""
! self.count = 0
! def GetParent(self):
! # return a folder object with the parent, or None
! raise NotImplementedError
! def GetMessageGenerator(self, folder, only_filter_candidates = True):
! # Return a generator of MsgStoreMsg objects for the folder
! raise NotImplementedError
!
! class MsgStoreMsg:
! def __init__(self):
! pass
! def GetEmailPackageObject(self):
! # Return a "read-only" Python email package object
! # "read-only" in that changes will never be reflected to the real store.
! raise NotImplementedError
! def SetField(self, name, value):
! # Abstractly set a user field name/id to a field value.
! # User field is for the user to see - status/internal fields
! # should get their own methods
! raise NotImplementedError
! def GetSubject(self):
! # Get the subject - function as it may require a trip to the store!
! raise NotImplementedError
! def GetField(self, name):
! # Abstractly get a user field name/id to a field value.
! raise NotImplementedError
! def Save(self):
! # Save changes after field changes.
! raise NotImplementedError
! def MoveTo(self, folder_id):
! # Move the message to a folder.
! raise NotImplementedError
! def CopyTo(self, folder_id):
! # Copy the message to a folder.
! raise NotImplementedError
! def IsFilterCandidate(self):
! # Return True if this is a message that should be checked for spam
! # Return False if it should be ignored (eg, user-composed message,
! # undeliverable report, meeting request etc.
! raise NotImplementedError
!
! # Our MAPI implementation
! import warnings
! if sys.version_info >= (2, 3):
! # sick off the new hex() warnings!
! warnings.filterwarnings("ignore", category=FutureWarning, append=1)
!
from win32com.client import Dispatch, constants
from win32com.mapi import mapi, mapiutil
--- 10,14 ----
True, False = 1, 0
! # MAPI imports etc.
from win32com.client import Dispatch, constants
from win32com.mapi import mapi, mapiutil
***************
*** 96,99 ****
--- 16,20 ----
import pythoncom
+ # Additional MAPI constants we dont have in Python
MESSAGE_MOVE = 0x1 # from MAPIdefs.h
MSGFLAG_READ = 0x1 # from MAPIdefs.h
***************
*** 110,113 ****
--- 31,96 ----
USE_DEFERRED_ERRORS = mapi.MAPI_DEFERRED_ERRORS # or set to zero to see what changes
+ #import warnings
+ #if sys.version_info >= (2, 3):
+ # # sick off the new hex() warnings!
+ # warnings.filterwarnings("ignore", category=FutureWarning, append=1)
+
+ # Nod to our automated test suite. Currently supports a hack so our test
+ # message is filtered, and also for raising exceptions at key times.
+ # see tester.py for more details.
+ test_suite_running = False
+ test_suite_failure_request = None
+ test_suite_failure = None
+ # Sometimes the test suite will request that we simulate MAPI errors.
+ def help_test_suite(checkpoint_name):
+ if test_suite_running and \
+ test_suite_failure_request == checkpoint_name:
+ raise test_suite_failure[0], test_suite_failure[1]
+
+ # Exceptions raised by this module. Raw MAPI exceptions should never
+ # be raised to the caller.
+ class MsgStoreException(Exception):
+ def __init__(self, mapi_exception, extra_msg = None):
+ self.mapi_exception = mapi_exception
+ self.extra_msg = extra_msg
+ Exception.__init__(self, mapi_exception, extra_msg)
+ def __str__(self):
+ try:
+ return "%s: %s" % (self.__class__.__name__,
+ GetCOMExceptionString(self.mapi_exception))
+ except:
+ print "Error __str__"
+ import traceback
+ traceback.print_exc()
+
+ # Exception raised when you attempt to get a message or folder that doesn't
+ # exist. Usually means you are querying an ID that *was* valid, but has
+ # since been moved or deleted.
+ # Note you may get this exception "getting" objects (such as messages or
+ # folders), or accessing properties once the object was created (the message
+ # may be moved under us at any time)
+ class NotFoundException(MsgStoreException):
+ pass
+
+ # Exception raised when you try and modify a "read only" object.
+ # Only currently examples are Hotmail and IMAP folders.
+ class ReadOnlyException(MsgStoreException):
+ pass
+
+ # Utility functions for exceptions. Convert a COM exception to the best
+ # manager exception.
+ def MsgStoreExceptionFromCOMException(com_exc):
+ if IsNotFoundCOMException(com_exc):
+ return NotFoundException(com_exc)
+ if IsReadOnlyCOMException(com_exc):
+ return ReadOnlyException(com_exc)
+ return MsgStoreException(com_exc)
+
+ # Build a reasonable string from a COM exception tuple
+ def GetCOMExceptionString(exc_val):
+ hr, msg, exc, arg_err = exc_val
+ err_string = mapiutil.GetScodeString(hr)
+ return "Exception 0x%x (%s): %s" % (hr, err_string, msg)
+
# Does this exception probably mean "object not found"?
def IsNotFoundCOMException(exc_val):
***************
*** 121,130 ****
return hr == mapi.MAPI_E_FAILONEPROVIDER
! def GetCOMExceptionString(exc_val):
! hr, msg, exc, arg_err = exc_val
! err_string = mapiutil.GetScodeString(hr)
! return "Exception 0x%x (%s): %s" % (hr, err_string, msg)
! def ReportMAPIError(manager, what, exc_type, exc_val):
hr, exc_msg, exc, arg_err = exc_val
if hr == mapi.MAPI_E_TABLE_TOO_BIG:
--- 104,115 ----
return hr == mapi.MAPI_E_FAILONEPROVIDER
! def IsReadOnlyCOMException(exc_val):
! # This seems to happen for IMAP mails (0x800cccd3)
! # and also for hotmail messages (0x8004dff7)
! known_failure_codes = -2146644781, -2147164169
! return exc_val[0] in known_failure_codes
!
! def ReportMAPIError(manager, what, exc_val):
hr, exc_msg, exc, arg_err = exc_val
if hr == mapi.MAPI_E_TABLE_TOO_BIG:
***************
*** 142,146 ****
manager.ReportErrorOnce(err_msg)
! class MAPIMsgStore(MsgStore):
def __init__(self, outlook = None):
self.outlook = outlook
--- 127,137 ----
manager.ReportErrorOnce(err_msg)
! # Our objects.
! class MAPIMsgStore:
! # Stash exceptions in the class for ease of use by consumers.
! MsgStoreException = MsgStoreException
! NotFoundException = NotFoundException
! ReadOnlyException = ReadOnlyException
!
def __init__(self, outlook = None):
self.outlook = outlook
***************
*** 298,355 ****
def GetFolder(self, folder_id):
# Return a single folder given the ID.
! try:
! # See if this is an Outlook folder item
! sid = mapi.BinFromHex(folder_id.StoreID)
! eid = mapi.BinFromHex(folder_id.EntryID)
! folder_id = sid, eid
! except AttributeError:
! # No 'EntryID'/'StoreID' properties - a 'normal' ID
! folder_id = self.NormalizeID(folder_id)
! except pythoncom.com_error, details:
! if IsNotFoundCOMException(details):
! print "Unable to open folder '%r'" \
! "- the folder was not found" % folder_id
! else:
! print "Unexpected MAPI error opening folder"
! print GetCOMExceptionString(details)
! return None
! # Also catch COM exceptions opening the folder and table.
! try:
folder = self._OpenEntry(folder_id)
table = folder.GetContentsTable(0)
! except pythoncom.com_error, details:
! # We will ignore *all* such errors for the time
! # being, but warn for results we don't know about.
! if not IsNotFoundCOMException(details):
! print "WARNING: Unexpected MAPI error opening folder"
! print GetCOMExceptionString(details)
! return None
! # Ensure we have a long-term ID.
! rc, props = folder.GetProps( (PR_ENTRYID, PR_DISPLAY_NAME_A), 0)
! folder_id = folder_id[0], props[0][1]
! return MAPIMsgStoreFolder(self, folder_id, props[1][1],
table.GetRowCount(0))
def GetMessage(self, message_id):
# Return a single message given either the ID, or an Outlook
# message representing the object.
! try:
! eid = mapi.BinFromHex(message_id.EntryID)
! sid = mapi.BinFromHex(message_id.Parent.StoreID)
! message_id = sid, eid
! except AttributeError:
! # No 'EntryID'/'StoreID' properties - a 'normal' ID
! message_id = self.NormalizeID(message_id)
except pythoncom.com_error, details:
! if IsNotFoundCOMException(details):
! print "Unable to open message '%r'" \
! "- the message was not found" % message_id
! else:
! print "Unexpected MAPI error opening message"
! print GetCOMExceptionString(details)
! return None
! mapi_object = self._OpenEntry(message_id)
! hr, data = mapi_object.GetProps(MAPIMsgStoreMsg.message_init_props,0)
! return MAPIMsgStoreMsg(self, data)
def YieldReceiveFolders(self, msg_class = "IPM.Note"):
--- 289,327 ----
def GetFolder(self, folder_id):
# Return a single folder given the ID.
! try: # catch all MAPI errors
! try:
! # See if this is an Outlook folder item
! sid = mapi.BinFromHex(folder_id.StoreID)
! eid = mapi.BinFromHex(folder_id.EntryID)
! folder_id = sid, eid
! except AttributeError:
! # No 'EntryID'/'StoreID' properties - a 'normal' ID
! folder_id = self.NormalizeID(folder_id)
folder = self._OpenEntry(folder_id)
table = folder.GetContentsTable(0)
! # Ensure we have a long-term ID.
! rc, props = folder.GetProps( (PR_ENTRYID, PR_DISPLAY_NAME_A), 0)
! folder_id = folder_id[0], props[0][1]
! return MAPIMsgStoreFolder(self, folder_id, props[1][1],
table.GetRowCount(0))
+ except pythoncom.com_error, details:
+ raise MsgStoreExceptionFromCOMException(details)
def GetMessage(self, message_id):
# Return a single message given either the ID, or an Outlook
# message representing the object.
! try: # catch all MAPI exceptions.
! try:
! eid = mapi.BinFromHex(message_id.EntryID)
! sid = mapi.BinFromHex(message_id.Parent.StoreID)
! message_id = sid, eid
! except AttributeError:
! # No 'EntryID'/'StoreID' properties - a 'normal' ID
! message_id = self.NormalizeID(message_id)
! mapi_object = self._OpenEntry(message_id)
! hr, data = mapi_object.GetProps(MAPIMsgStoreMsg.message_init_props,0)
! return MAPIMsgStoreMsg(self, data)
except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def YieldReceiveFolders(self, msg_class = "IPM.Note"):
***************
*** 451,455 ****
return html or ''
! class MAPIMsgStoreFolder(MsgStoreMsg):
def __init__(self, msgstore, id, name, count):
self.msgstore = msgstore
--- 423,427 ----
return html or ''
! class MAPIMsgStoreFolder:
def __init__(self, msgstore, id, name, count):
self.msgstore = msgstore
***************
*** 558,561 ****
--- 530,535 ----
# Our restriction helped, but may not have filtered
# every message we don't want to touch.
+ # Note no exception will be raised below if the message is
+ # moved under us, as we don't need to access any properties.
msg = MAPIMsgStoreMsg(self.msgstore, row)
if not only_filter_candidates or msg.IsFilterCandidate():
***************
*** 596,599 ****
--- 570,575 ----
break
for row in rows:
+ # Note no exception will be raised below if the message is
+ # moved under us, as we don't need to access any properties.
msg = MAPIMsgStoreMsg(self.msgstore, row)
if msg.IsFilterCandidate():
***************
*** 615,620 ****
return self._FolderFromMAPIFolder(ret)
!
! class MAPIMsgStoreMsg(MsgStoreMsg):
# All the properties we must initialize a message with.
# These include all the IDs we need, parent IDs, any properties needed
--- 591,595 ----
return self._FolderFromMAPIFolder(ret)
! class MAPIMsgStoreMsg:
# All the properties we must initialize a message with.
# These include all the IDs we need, parent IDs, any properties needed
***************
*** 713,716 ****
--- 688,694 ----
# it manages given it is an external server, and as far as I can tell,
# this does not appear in the headers.
+ if test_suite_running:
+ # While the test suite is running, we *only* filter test msgs.
+ return self.subject == "SpamBayes addin auto-generated test message"
class_check = self.msgclass.lower()
for check in "ipm.note", "ipm.anti-virus":
***************
*** 721,725 ****
return False
# Must match msg class to get here.
! return self.was_received or test_suite_running
def _GetPotentiallyLargeStringProp(self, prop_id, row):
--- 699,703 ----
return False
# Must match msg class to get here.
! return self.was_received
def _GetPotentiallyLargeStringProp(self, prop_id, row):
***************
*** 848,852 ****
def _EnsureObject(self):
if self.mapi_object is None:
! self.mapi_object = self.msgstore._OpenEntry(self.id)
def GetEmailPackageObject(self, strip_mime_headers=True):
--- 826,834 ----
def _EnsureObject(self):
if self.mapi_object is None:
! try:
! help_test_suite("MAPIMsgStoreMsg._EnsureObject")
! self.mapi_object = self.msgstore._OpenEntry(self.id)
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def GetEmailPackageObject(self, strip_mime_headers=True):
***************
*** 951,977 ****
# XXX Outlook to define a custom Integer field of the same name.
self._EnsureObject()
- if type(prop) != type(0):
- props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
- propIds = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE)
- type_tag = _MapiTypeMap.get(type(val))
- if type_tag is None:
- raise ValueError, "Don't know what to do with '%r' ('%s')" % (
- val, type(val))
- prop = PROP_TAG(type_tag, PROP_ID(propIds[0]))
- if val is None:
- # Delete the property
- self.mapi_object.DeleteProps((prop,))
- else:
- self.mapi_object.SetProps(((prop,val),))
- self.dirty = True
-
- def GetField(self, prop, raise_errors = False):
try:
! self._EnsureObject()
except pythoncom.com_error, details:
! if not IsNotFoundCOMException(details):
! print "ERROR: Could not open an object to fetch a field"
! print details
! return None
if type(prop) != type(0):
props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
--- 933,958 ----
# XXX Outlook to define a custom Integer field of the same name.
self._EnsureObject()
try:
! if type(prop) != type(0):
! props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
! propIds = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE)
! type_tag = _MapiTypeMap.get(type(val))
! if type_tag is None:
! raise ValueError, "Don't know what to do with '%r' ('%s')" % (
! val, type(val))
! prop = PROP_TAG(type_tag, PROP_ID(propIds[0]))
! help_test_suite("MAPIMsgStoreMsg.SetField")
! if val is None:
! # Delete the property
! self.mapi_object.DeleteProps((prop,))
! else:
! self.mapi_object.SetProps(((prop,val),))
! self.dirty = True
except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
!
! def GetField(self, prop):
! # xxx - still raise_errors?
! self._EnsureObject()
if type(prop) != type(0):
props = ( (mapi.PS_PUBLIC_STRINGS, prop), )
***************
*** 989,996 ****
return None
return val
! except:
! if raise_errors:
! raise
! return None
def GetReadState(self):
--- 970,975 ----
return None
return val
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def GetReadState(self):
***************
*** 999,1046 ****
def SetReadState(self, is_read):
! self._EnsureObject()
! # always try and clear any pending delivery reports of read/unread
! if is_read:
! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|SUPPRESS_RECEIPT)
! else:
! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|CLEAR_READ_FLAG)
! if __debug__:
! if self.GetReadState() != is_read:
! print "MAPI SetReadState appears to have failed to change the message state"
! print "Requested set to %s but the MAPI field after was %r" % \
! (is_read, self.GetField(PR_MESSAGE_FLAGS))
def Save(self):
assert self.dirty, "asking me to save a clean message!"
! # There are some known exceptions that can be raised by IMAP and hotmail
! # For now, we just let the caller handle all errors, and manually
! # reset the dirty flag. Only current caller is filter.py
! # There are also some issues with the "unread flag" that fiddling this
! # save code may fix.
! # It seems that *not* specifying mapi.MAPI_DEFERRED_ERRORS solves alot
! # of said problems though! So we don't!
! self.mapi_object.SaveChanges(mapi.KEEP_OPEN_READWRITE)
! self.dirty = False
def _DoCopyMove(self, folder, isMove):
assert not self.dirty, \
"asking me to move a dirty message - later saves will fail!"
! dest_folder = self.msgstore._OpenEntry(folder.id)
! source_folder = self.msgstore._OpenEntry(self.folder_id)
! flags = 0
! if isMove: flags |= MESSAGE_MOVE
! eid = self.id[1]
! source_folder.CopyMessages((eid,),
! None,
! dest_folder,
! 0,
! None,
! flags)
! # At this stage, I think we have lost meaningful ID etc values
! # Set everything to None to make it clearer what is wrong should
! # this become an issue. We would need to re-fetch the eid of
! # the item, and set the store_id to the dest folder.
! self.id = None
! self.folder_id = None
def MoveTo(self, folder):
--- 978,1032 ----
def SetReadState(self, is_read):
! try:
! self._EnsureObject()
! # always try and clear any pending delivery reports of read/unread
! help_test_suite("MAPIMsgStoreMsg.SetReadState")
! if is_read:
! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|SUPPRESS_RECEIPT)
! else:
! self.mapi_object.SetReadFlag(USE_DEFERRED_ERRORS|CLEAR_READ_FLAG)
! if __debug__:
! if self.GetReadState() != is_read:
! print "MAPI SetReadState appears to have failed to change the message state"
! print "Requested set to %s but the MAPI field after was %r" % \
! (is_read, self.GetField(PR_MESSAGE_FLAGS))
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def Save(self):
assert self.dirty, "asking me to save a clean message!"
! # It seems that *not* specifying mapi.MAPI_DEFERRED_ERRORS solves a lot
! # problems! So we don't!
! try:
! help_test_suite("MAPIMsgStoreMsg.Save")
! self.mapi_object.SaveChanges(mapi.KEEP_OPEN_READWRITE)
! self.dirty = False
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def _DoCopyMove(self, folder, isMove):
assert not self.dirty, \
"asking me to move a dirty message - later saves will fail!"
! try:
! dest_folder = self.msgstore._OpenEntry(folder.id)
! source_folder = self.msgstore._OpenEntry(self.folder_id)
! flags = 0
! if isMove: flags |= MESSAGE_MOVE
! eid = self.id[1]
! help_test_suite("MAPIMsgStoreMsg._DoCopyMove")
! source_folder.CopyMessages((eid,),
! None,
! dest_folder,
! 0,
! None,
! flags)
! # At this stage, I think we have lost meaningful ID etc values
! # Set everything to None to make it clearer what is wrong should
! # this become an issue. We would need to re-fetch the eid of
! # the item, and set the store_id to the dest folder.
! self.id = None
! self.folder_id = None
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def MoveTo(self, folder):
***************
*** 1053,1068 ****
# user. Any errors are re-raised so the caller can degrade gracefully if
# necessary.
def MoveToReportingError(self, manager, folder):
try:
self.MoveTo(folder)
! except pythoncom.com_error, details:
! ReportMAPIError(manager, "Moving a message", pythoncom.com_error, details)
! raise
def CopyToReportingError(self, manager, folder):
try:
self.MoveTo(folder)
! except pythoncom.com_error, details:
! ReportMAPIError(manager, "Copying a message", pythoncom.com_error, details)
! raise
def GetFolder(self):
--- 1039,1055 ----
# user. Any errors are re-raised so the caller can degrade gracefully if
# necessary.
+ # XXX - not too happy with these - they should go, and the caller should
+ # handle (especially now that we work exclusively with exceptions from
+ # this module.
def MoveToReportingError(self, manager, folder):
try:
self.MoveTo(folder)
! except MsgStoreException, details:
! ReportMAPIError(manager, "Moving a message", details.mapi_exception)
def CopyToReportingError(self, manager, folder):
try:
self.MoveTo(folder)
! except MsgStoreException, details:
! ReportMAPIError(manager, "Copying a message", details.mapi_exception)
def GetFolder(self):
***************
*** 1074,1088 ****
def RememberMessageCurrentFolder(self):
self._EnsureObject()
! folder = self.GetFolder()
! props = ( (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderStoreID"),
! (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderID")
! )
! resolve_ids = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE)
! prop_ids = PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[0])), \
! PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[1]))
!
! prop_tuples = (prop_ids[0],folder.id[0]), (prop_ids[1],folder.id[1])
! self.mapi_object.SetProps(prop_tuples)
! self.dirty = True
def GetRememberedFolder(self):
--- 1061,1078 ----
def RememberMessageCurrentFolder(self):
self._EnsureObject()
! try:
! folder = self.GetFolder()
! props = ( (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderStoreID"),
! (mapi.PS_PUBLIC_STRINGS, "SpamBayesOriginalFolderID")
! )
! resolve_ids = self.mapi_object.GetIDsFromNames(props, mapi.MAPI_CREATE)
! prop_ids = PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[0])), \
! PROP_TAG( PT_BINARY, PROP_ID(resolve_ids[1]))
!
! prop_tuples = (prop_ids[0],folder.id[0]), (prop_ids[1],folder.id[1])
! self.mapi_object.SetProps(prop_tuples)
! self.dirty = True
! except pythoncom.com_error, details:
! raise MsgStoreExceptionFromCOMException(details)
def GetRememberedFolder(self):
***************
*** 1100,1103 ****
--- 1090,1094 ----
(store_tag, store_id), (eid_tag, eid) = data
folder_id = mapi.HexFromBin(store_id), mapi.HexFromBin(eid)
+ help_test_suite("MAPIMsgStoreMsg.GetRememberedFolder")
return self.msgstore.GetFolder(folder_id)
except:
Index: tester.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** tester.py 1 Sep 2003 05:35:58 -0000 1.16
--- tester.py 4 Sep 2003 12:14:12 -0000 1.17
***************
*** 16,19 ****
--- 16,23 ----
import rfc822
import cStringIO
+ import threading
+
+ from win32com.mapi import mapi, mapiutil
+ import pythoncom
HAM="ham"
***************
*** 29,37 ****
--- 33,46 ----
raise TestFailure(msg)
+ filter_event = threading.Event()
+
def WaitForFilters():
import pythoncom
# Must wait longer than normal, so when run with a timer we still work.
+ filter_event.clear()
for i in range(500):
pythoncom.PumpWaitingMessages()
+ if filter_event.isSet():
+ break
sleep(0.01)
***************
*** 371,374 ****
--- 380,384 ----
def run_tests(manager):
+ "Filtering tests"
driver = Driver(manager)
manager.Save() # necessary after a full retrain
***************
*** 384,408 ****
def run_filter_tests(manager):
# setup config to save info with the message, and test
! print "*" * 10, "Running tests with save_spam_info=True, timer off"
! manager.config.experimental.timer_start_delay = 0
! manager.config.experimental.timer_interval = 0
! manager.config.filter.save_spam_info = True
! manager.addin.FiltersChanged() # to ensure correct filtler in place
! run_tests(manager)
! # do it again with the same config, just to prove we can.
! print "*" * 10, "Running them again with save_spam_info=True"
! run_tests(manager)
! # enable the timer.
! manager.config.experimental.timer_start_delay = 1000
! manager.config.experimental.timer_interval = 500
! manager.addin.FiltersChanged() # to switch to timer based filters.
! print "*" * 10, "Running them again with save_spam_info=True, and timer enabled"
! run_tests(manager)
! # and with save_spam_info False.
! print "*" * 10, "Running tests with save_spam_info=False"
! manager.config.filter.save_spam_info = False
! run_tests(manager)
! print "*" * 10, "Filtering tests completed successfully."
def run_nonfilter_tests(manager):
# And now some other 'sanity' checks.
--- 394,441 ----
def run_filter_tests(manager):
# setup config to save info with the message, and test
! apply_with_new_config(manager,
! {"Filter.timer_enabled": False,
! "Filter.save_spam_info" : True,
! },
! run_tests, manager)
!
! apply_with_new_config(manager,
! {"Filter.timer_enabled": True,
! "Filter.save_spam_info" : True,
! },
! run_tests, manager)
! apply_with_new_config(manager,
! {"Filter.timer_enabled": False,
! "Filter.save_spam_info" : False,
! },
! run_tests, manager)
!
! apply_with_new_config(manager,
! {"Filter.timer_enabled": True,
! "Filter.save_spam_info" : False,
! },
! run_tests, manager)
+ def apply_with_new_config(manager, new_config_dict, func, *args):
+ old_config = {}
+ friendly_opts = []
+ for name, val in new_config_dict.items():
+ sect_name, opt_name = name.split(".")
+ old_config[sect_name, opt_name] = manager.options.get(sect_name, opt_name)
+ manager.options.set(sect_name, opt_name, val)
+ friendly_opts.append("%s=%s" % (name, val))
+ manager.addin.FiltersChanged() # to ensure correct filtler in place
+ try:
+ test_name = getattr(func, "__doc__", None)
+ if not test_name: test_name = func.__name__
+ print "*" * 10, "Running '%s' with %s" % (test_name, ", ".join(friendly_opts))
+ func(*args)
+ finally:
+ for (sect_name, opt_name), val in old_config.items():
+ manager.options.set(sect_name, opt_name, val)
+
+ ###############################################################################
+ # "Non-filter" tests are those that don't require us to create messages and
+ # see them get filtered.
def run_nonfilter_tests(manager):
# And now some other 'sanity' checks.
***************
*** 425,429 ****
# reported.
num_looked += 1
! if num_looked % 500 == 0: print " (scanned", num_looked, "messages...)"
if not message.IsFilterCandidate() and \
message.msgclass.lower().startswith("ipm.note"):
--- 458,462 ----
# reported.
num_looked += 1
! if num_looked % 500 == 0: print " scanned", num_looked, "messages..."
if not message.IsFilterCandidate() and \
message.msgclass.lower().startswith("ipm.note"):
***************
*** 449,466 ****
--- 482,601 ----
msgstore.test_suite_running = True
+ ###############################################################################
+ # "Failure" tests - execute some tests while provoking the msgstore to simulate
+ # various MAPI errors. Although not complete, it does help exercise our code
+ # paths through the code.
+ def _restore_mapi_failure():
+ import msgstore
+ msgstore.test_suite_failure = None
+ msgstore.test_suite_failure_request = None
+
+ def _setup_for_mapi_failure(checkpoint, hr):
+ import msgstore
+ assert msgstore.test_suite_running, "msgstore should already know its running"
+ assert not msgstore.test_suite_failure, "should already have torn down previous failure"
+ msgstore.test_suite_failure = pythoncom.com_error, \
+ (hr, "testsuite generated error", None, -1)
+ msgstore.test_suite_failure_request = checkpoint
+
+ def _setup_mapi_notfound_failure(checkpoint):
+ _setup_for_mapi_failure(checkpoint, mapi.MAPI_E_NOT_FOUND)
+
+ def _do_single_failure_ham_test(driver, checkpoint, hr):
+ _do_single_failure_test(driver, True, checkpoint, hr)
+
+ def _do_single_failure_spam_test(driver, checkpoint, hr):
+ _do_single_failure_test(driver, False, checkpoint, hr)
+
+ def _do_single_failure_test(driver, is_ham, checkpoint, hr):
+ print "-> Testing MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr),
+ checkpoint)
+ # message moved after we have ID, but before opening.
+ folder = driver.folder_watch
+ if is_ham:
+ msg, words = driver.CreateTestMessageInFolder(HAM, folder)
+ else:
+ msg, words = driver.CreateTestMessageInFolder(SPAM, folder)
+ try:
+ _setup_for_mapi_failure(checkpoint, hr)
+ try:
+ # sleep to ensure filtering.
+ WaitForFilters()
+ finally:
+ _restore_mapi_failure()
+ if driver.FindTestMessage(folder) is None:
+ TestFailed("We appear to have filtered a message even though we forced 'not found' failure")
+ finally:
+ print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr),
+ checkpoint)
+ if msg is not None:
+ msg.Delete()
+
+ def do_failure_tests(manager):
+ # We setup msgstore to fail for us, then try a few tests. The idea is to
+ # ensure we gracefully degrade in these failures.
+ # We set verbosity to min of 1, as this helps us see how the filters handle
+ # the errors.
+ driver = Driver(manager)
+ driver.CleanAllTestMessages()
+ old_verbose = manager.verbose
+ manager.verbose = max(1, old_verbose)
+ try:
+ _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg._EnsureObject", mapi.MAPI_E_NOT_FOUND)
+ _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg.SetField", -2146644781)
+ _do_single_failure_ham_test(driver, "MAPIMsgStoreMsg.Save", -2146644781)
+ # SetReadState???
+ _do_single_failure_spam_test(driver, "MAPIMsgStoreMsg._DoCopyMove", mapi.MAPI_E_TABLE_TOO_BIG)
+ finally:
+ manager.verbose = old_verbose
+
+ def run_failure_tests(manager):
+ "Forced MAPI failure tests"
+ apply_with_new_config(manager,
+ {"Filter.timer_enabled": True,
+ },
+ do_failure_tests, manager)
+ apply_with_new_config(manager,
+ {"Filter.timer_enabled": False,
+ },
+ do_failure_tests, manager)
+
+ def filter_message_with_event(msg, mgr, all_actions=True):
+ import filter
+ try:
+ return filter._original_filter_message(msg, mgr, all_actions)
+ finally:
+ filter_event.set()
+
def test(manager):
import msgstore
from dialogs import SetWaitCursor
SetWaitCursor(1)
+
+ import filter
+ if "_original_filter_message" not in filter.__dict__:
+ filter._original_filter_message = filter.filter_message
+ filter.filter_message = filter_message_with_event
+
try: # restore the plugin config at exit.
+ assert not msgstore.test_suite_running, "already running??"
msgstore.test_suite_running = True
+ assert not manager.test_suite_running, "already running??"
+ manager.test_suite_running = True
run_nonfilter_tests(manager)
# filtering tests take alot of time - do them last.
run_filter_tests(manager)
+ run_failure_tests(manager)
+ print "*" * 20
+ print "Test suite finished without error!"
+ print "*" * 20
finally:
+ print "Restoring standard configuration..."
# Always restore configuration to how we started.
msgstore.test_suite_running = False
+ manager.test_suite_running = False
manager.LoadConfig()
manager.addin.FiltersChanged() # restore original filters.
+ manager.addin.ProcessMissedMessages()
SetWaitCursor(0)
From richiehindle at users.sourceforge.net Thu Sep 4 15:30:48 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Thu Sep 4 17:30:52 2003
Subject: [Spambayes-checkins] spambayes/spambayes __init__.py,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv8404
Modified Files:
__init__.py
Log Message:
Uprev to 1.0a5.
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/__init__.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** __init__.py 7 Jul 2003 00:19:31 -0000 1.5
--- __init__.py 4 Sep 2003 21:30:46 -0000 1.6
***************
*** 1,3 ****
# package marker.
! __version__ = '1.0a4'
--- 1,3 ----
# package marker.
! __version__ = '1.0a5'
From mhammond at users.sourceforge.net Thu Sep 4 05:59:07 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 4 18:18:46 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
opt_processors.py, 1.12, 1.13
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv18964
Modified Files:
opt_processors.py
Log Message:
Remove stray 'print'
Index: opt_processors.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/opt_processors.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** opt_processors.py 26 Aug 2003 06:25:33 -0000 1.12
--- opt_processors.py 4 Sep 2003 11:59:05 -0000 1.13
***************
*** 190,194 ****
slider_pos = float(slider_pos) * self.max_val / self.ticks
str_val = str(slider_pos)
- print "Slider wants to set to", str_val
edit = self.GetControl()
win32gui.SendMessage(edit, win32con.WM_SETTEXT, 0, str_val)
--- 190,193 ----
From anadelonbrin at users.sourceforge.net Thu Sep 4 17:29:49 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 19:29:54 2003
Subject: [Spambayes-checkins] website quotes.ht,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv29971
Modified Files:
quotes.ht
Log Message:
More nice things about us, this time from the TechTV article.
(Including a positive comment about the documentation!)
Index: quotes.ht
===================================================================
RCS file: /cvsroot/spambayes/website/quotes.ht,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** quotes.ht 16 Aug 2003 13:39:17 -0000 1.4
--- quotes.ht 4 Sep 2003 23:29:47 -0000 1.5
***************
*** 51,54 ****
--- 51,61 ----
+
+ I have to say I'm very happy with its performance...The documentation
+ on the SpamBayes site is very good...It is very much worth the time to
+ train and use SpamBayes.
+ Chris DiBona in a TechTV article.
+
+
What we are saying about us
From mhammond at users.sourceforge.net Thu Sep 4 05:59:37 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 4 19:46:55 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/installer
spambayes_addin.iss, 1.8, 1.9
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/installer
In directory sc8-pr-cvs1:/tmp/cvs-serv19031
Modified Files:
spambayes_addin.iss
Log Message:
'welcome.html' is now the 'readme' for the installer.
Index: spambayes_addin.iss
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** spambayes_addin.iss 9 Aug 2003 03:26:16 -0000 1.8
--- spambayes_addin.iss 4 Sep 2003 11:59:35 -0000 1.9
***************
*** 18,22 ****
Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver
Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
! Source: "dist\about.html"; DestDir: "{app}"; Flags: isreadme
[UninstallDelete]
--- 18,22 ----
Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver
Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
! Source: "dist\docs\welcome.html"; DestDir: "{app}"; Flags: isreadme
[UninstallDelete]
From ta-meyer at ihug.co.nz Fri Sep 5 13:05:31 2003
From: ta-meyer at ihug.co.nz (Tony Meyer)
Date: Thu Sep 4 20:05:42 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py, NONE,
1.1
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13031B74DA@its-xchg4.massey.ac.nz>
Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F130212AE19@its-xchg4.massey.ac.nz>
> Anyone know if something is up with the SourceForge CVS servers?
I've seen other reports of this as well. I don't know for sure, but I
imagine that it has something to do with sourceforge getting everything back
to the normal status. Their last email update said it was only days away,
so they're probably installing something and letting anon cvs hang for
periods without updating.
=Tony Meyer
From anadelonbrin at users.sourceforge.net Thu Sep 4 18:24:50 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 20:24:54 2003
Subject: [Spambayes-checkins] spambayes dbExpImp.py, 1.7, NONE hammie.py,
1.46, NONE hammiecli.py, 1.4, NONE hammiefilter.py, 1.19,
NONE hammiesrv.py, 1.11, NONE imapfilter.py, 1.54,
NONE mailsort.py, 1.7, NONE mboxtrain.py, 1.12,
NONE notesfilter.py, 1.3, NONE pop3proxy.py, 1.97,
NONE proxytee.py, 1.4, NONE smtpproxy.py, 1.15,
NONE unheader.py, 1.10, NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6818
Removed Files:
dbExpImp.py hammie.py hammiecli.py hammiefilter.py
hammiesrv.py imapfilter.py mailsort.py mboxtrain.py
notesfilter.py pop3proxy.py proxytee.py smtpproxy.py
unheader.py
Log Message:
Step one of the 1.0a6 shuffle. Remove all of the top-level scripts and
replace them with sb-* versions in the scripts directory. These are the
names that seemed to be the agreement on spambayes-dev - change
them quick if you don't like them ;)
Note for anyone reading this in the cvs history - for older history than
this, you will need to go to the script directory's parent's attic and look
at the old files. The names are the same once the sb- prefix is removed,
with the exception of:
sb-client was hammiecli
sb-filter was hammiefilter
sb-pop3dnd was sb-overkill
sb-server was pop3proxy
sb-upload was proxytee
sb-xmlrpcserver was hammiesrv
--- dbExpImp.py DELETED ---
--- hammie.py DELETED ---
--- hammiecli.py DELETED ---
--- hammiefilter.py DELETED ---
--- hammiesrv.py DELETED ---
--- imapfilter.py DELETED ---
--- mailsort.py DELETED ---
--- mboxtrain.py DELETED ---
--- notesfilter.py DELETED ---
--- pop3proxy.py DELETED ---
--- proxytee.py DELETED ---
--- smtpproxy.py DELETED ---
--- unheader.py DELETED ---
From richiehindle at users.sourceforge.net Thu Sep 4 15:35:31 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Thu Sep 4 20:35:33 2003
Subject: [Spambayes-checkins] website index.ht,1.23,1.24
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv9311
Modified Files:
index.ht
Log Message:
*Fifth* pre-release available.
Index: index.ht
===================================================================
RCS file: /cvsroot/spambayes/website/index.ht,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** index.ht 26 Aug 2003 08:08:57 -0000 1.23
--- index.ht 4 Sep 2003 21:35:29 -0000 1.24
***************
*** 6,10 ****
News
Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
! Fourth pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
--- 6,10 ----
News
Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
! Fifth pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
From anadelonbrin at users.sourceforge.net Thu Sep 4 18:24:50 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 20:42:29 2003
Subject: [Spambayes-checkins] spambayes/scripts sb-client.py, NONE,
1.1 sb-dbexpimp.py, NONE, 1.1 sb-filter.py, NONE,
1.1 sb-imapfilter.py, NONE, 1.1 sb-mailsort.py, NONE,
1.1 sb-mboxtrain.py, NONE, 1.1 sb-notesfilter.py, NONE,
1.1 sb-pop3dnd.py, NONE, 1.1 sb-server.py, NONE,
1.1 sb-smtpproxy.py, NONE, 1.1 sb-unheader.py, NONE,
1.1 sb-upload.py, NONE, 1.1 sb-xmlrpcserver.py, NONE,
1.1 sb-overkill.py, 1.1, NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv6818/scripts
Added Files:
sb-client.py sb-dbexpimp.py sb-filter.py sb-imapfilter.py
sb-mailsort.py sb-mboxtrain.py sb-notesfilter.py sb-pop3dnd.py
sb-server.py sb-smtpproxy.py sb-unheader.py sb-upload.py
sb-xmlrpcserver.py
Removed Files:
sb-overkill.py
Log Message:
Step one of the 1.0a6 shuffle. Remove all of the top-level scripts and
replace them with sb-* versions in the scripts directory. These are the
names that seemed to be the agreement on spambayes-dev - change
them quick if you don't like them ;)
Note for anyone reading this in the cvs history - for older history than
this, you will need to go to the script directory's parent's attic and look
at the old files. The names are the same once the sb- prefix is removed,
with the exception of:
sb-client was hammiecli
sb-filter was hammiefilter
sb-pop3dnd was sb-overkill
sb-server was pop3proxy
sb-upload was proxytee
sb-xmlrpcserver was hammiesrv
--- NEW FILE: sb-client.py ---
#! /usr/bin/env python
"""A client for hammiesrv.
Just feed it your mail on stdin, and it spits out the same message
with the spambayes score in a new X-Spambayes-Disposition header.
"""
import xmlrpclib
import sys
RPCBASE="http://localhost:65000"
def main():
msg = sys.stdin.read()
try:
x = xmlrpclib.ServerProxy(RPCBASE)
m = xmlrpclib.Binary(msg)
out = x.filter(m)
print out.data
except:
if __debug__:
import traceback
traceback.print_exc()
print msg
if __name__ == "__main__":
main()
--- NEW FILE: sb-dbexpimp.py ---
#! /usr/bin/env python
"""dbExpImp.py - Bayes database export/import
Classes:
Abstract:
This utility has the primary function of exporting and importing
a spambayes database into/from a flat file. This is useful in a number
of scenarios.
Platform portability of database - flat files can be exported and
imported across platforms (winduhs and linux, for example)
Database implementation changes - databases can survive database
implementation upgrades or new database implementations. For example,
if a dbm implementation changes between python x.y and python x.y+1...
Database reorganization - an export followed by an import reorgs an
existing database, improving performance, at least in
some database implementations
Database sharing - it is possible to distribute particular databases
for research purposes, database sharing purposes, or for new users to
have a 'seed' database to start with.
Database merging - multiple databases can be merged into one quite easily
by simply not specifying -n on an import. This will add the two database
nham and nspams together (assuming the two databases do not share corpora)
and for wordinfo conflicts, will add spamcount and hamcount together.
Spambayes software release migration - an export can be executed before
a release upgrade, as part of the installation script. Then, after the
new software is installed, an import can be executed, which will
effectively preserve existing training. This eliminates the need for
retraining every time a release is installed.
Others? I'm sure I haven't thought of everything...
Usage:
dbExpImp [options]
options:
-e : export
-i : import
-v : verbose mode (some additional diagnostic messages)
-f: FN : flat file to export to or import from
-d: FN : name of pickled database file to use
-D: FN : name of dbm database file to use
-m : merge import into an existing database file. This is
meaningful only for import. If omitted, a new database
file will be created. If specified, the imported
wordinfo will be merged into an existing database.
Run dbExpImp -h for more information.
-h : help
Examples:
Export pickled mybayes.db into mybayes.db.export as a csv flat file
dbExpImp -e -d mybayes.db -f mybayes.db.export
Import mybayes.eb.export into a new DBM mybayes.db
dbExpImp -i -D mybayes.db -f mybayes.db.export
Export, then import (reorganize) new pickled mybayes.db
dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export
Convert a bayes database from pickle to DBM
dbExpImp -e -d abayes.db -f abayes.export
dbExpImp -i -D abayes.db -f abayes.export
Create a new database (newbayes.db) from two
databases (abayes.db, bbayes.db)
dbExpImp -e -d abayes.db -f abayes.export
dbExpImp -e -d bbayes.db -f bbayes.export
dbExpImp -i -d newbayes.db -f abayes.export
dbExpImp -i -m -d newbayes.db -f bbayes.export
To Do:
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tim Stone "
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import spambayes.storage
from spambayes.Options import options
import sys, os, getopt, errno, re
import urllib
def runExport(dbFN, useDBM, outFN):
if useDBM:
bayes = spambayes.storage.DBDictClassifier(dbFN)
words = bayes.db.keys()
words.remove(bayes.statekey)
else:
bayes = spambayes.storage.PickledClassifier(dbFN)
words = bayes.wordinfo.keys()
try:
fp = open(outFN, 'w')
except IOError, e:
if e.errno != errno.ENOENT:
raise
nham = bayes.nham;
nspam = bayes.nspam;
print "Exporting database %s to file %s" % (dbFN, outFN)
print "Database has %s ham, %s spam, and %s words" \
% (nham, nspam, len(words))
fp.write("%s,%s,\n" % (nham, nspam))
for word in words:
wi = bayes._wordinfoget(word)
hamcount = wi.hamcount
spamcount = wi.spamcount
word = urllib.quote(word)
fp.write("%s`%s`%s`\n" % (word, hamcount, spamcount))
fp.close()
def runImport(dbFN, useDBM, newDBM, inFN):
if newDBM:
try:
os.unlink(dbFN)
except OSError, e:
if e.errno != 2: # errno.
raise
try:
os.unlink(dbFN+".dat")
except OSError, e:
if e.errno != 2: # errno.
raise
try:
os.unlink(dbFN+".dir")
except OSError, e:
if e.errno != 2: # errno.
raise
if useDBM:
bayes = spambayes.storage.DBDictClassifier(dbFN)
else:
bayes = spambayes.storage.PickledClassifier(dbFN)
try:
fp = open(inFN, 'r')
except IOError, e:
if e.errno != errno.ENOENT:
raise
nline = fp.readline()
(nham, nspam, junk) = re.split(',', nline)
if newDBM:
bayes.nham = int(nham)
bayes.nspam = int(nspam)
else:
bayes.nham += int(nham)
bayes.nspam += int(nspam)
if newDBM:
impType = "Importing"
else:
impType = "Merging"
print "%s database %s using file %s" % (impType, dbFN, inFN)
lines = fp.readlines()
for line in lines:
(word, hamcount, spamcount, junk) = re.split('`', line)
word = urllib.unquote(word)
try:
wi = bayes.wordinfo[word]
except KeyError:
wi = bayes.WordInfoClass()
wi.hamcount += int(hamcount)
wi.spamcount += int(spamcount)
bayes._wordinfoset(word, wi)
fp.close()
print "Storing database, please be patient. Even moderately large"
print "databases may take a very long time to store."
bayes.store()
print "Finished storing database"
if useDBM:
words = bayes.db.keys()
words.remove(bayes.statekey)
else:
words = bayes.wordinfo.keys()
print "Database has %s ham, %s spam, and %s words" \
% (bayes.nham, bayes.nspam, len(words))
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], 'iehmvd:D:f:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
usePickle = False
useDBM = False
newDBM = True
dbFN = None
flatFN = None
exp = False
imp = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
dbFN = arg
elif opt == '-D':
useDBM = True
dbFN = arg
elif opt == '-f':
flatFN = arg
elif opt == '-e':
exp = True
elif opt == '-i':
imp = True
elif opt == '-m':
newDBM = False
elif opt == '-v':
options["globals", "verbose"] = True
if (dbFN and flatFN):
if exp:
runExport(dbFN, useDBM, flatFN)
if imp:
runImport(dbFN, useDBM, newDBM, flatFN)
else:
print >>sys.stderr, __doc__
--- NEW FILE: sb-filter.py ---
#!/usr/bin/env python
## A hammie front-end to make the simple stuff simple.
##
##
## The intent is to call this from procmail and its ilk like so:
##
## :0 fw
## | hammiefilter.py
##
## Then, you can set up your MUA to pipe ham and spam to it, one at a
## time, by calling it with either the -g or -s options, respectively.
##
## Author: Neale Pickett
##
"""Usage: %(program)s [OPTION]...
[OPTION] is one of:
-h
show usage and exit
-x
show some usage examples and exit
-d DBFILE
use database in DBFILE
-D PICKLEFILE
use pickle (instead of database) in PICKLEFILE
-n
create a new database
*+ -f
filter (default if no processing options are given)
*+ -t
[EXPERIMENTAL] filter and train based on the result (you must
make sure to untrain all mistakes later)
* -g
[EXPERIMENTAL] (re)train as a good (ham) message
* -s
[EXPERIMENTAL] (re)train as a bad (spam) message
* -G
[EXPERIMENTAL] untrain ham (only use if you've already trained
this message)
* -S
[EXPERIMENTAL] untrain spam (only use if you've already trained
this message)
All options marked with '*' operate on stdin. Only those processing options
marked with '+' send a modified message to stdout.
"""
import os
import sys
import getopt
from spambayes import hammie, Options, mboxutils
# See Options.py for explanations of these properties
program = sys.argv[0]
example_doc = """_Examples_
filter a message on disk:
%(program)s < message
(re)train a message as ham:
%(program)s -g < message
(re)train a message as spam:
%(program)s -s < message
procmail recipe to filter and train in one step:
:0 fw
| %(program)s -t
mutt configuration. This binds the 'H' key to retrain the message as
ham, and prompt for a folder to move it to. The 'S' key retrains as
spam, and moves to a 'spam' folder.
XXX: add this
"""
def examples():
print example_doc % globals()
sys.exit(0)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
class HammieFilter(object):
def __init__(self):
options = Options.options
# This is a bit of a hack to counter the default for
# persistent_storage_file changing from ~/.hammiedb to hammie.db
# This will work unless a user:
# * had hammie.db as their value for persistent_storage_file, and
# * their config file was loaded by Options.py.
if options["Storage", "persistent_storage_file"] == \
options.default("Storage", "persistent_storage_file"):
options["Storage", "persistent_storage_file"] = \
"~/.hammiedb"
options.merge_files(['/etc/hammierc',
os.path.expanduser('~/.hammierc')])
self.dbname = options["Storage", "persistent_storage_file"]
self.dbname = os.path.expanduser(self.dbname)
self.usedb = options["Storage", "persistent_use_database"]
def newdb(self):
h = hammie.open(self.dbname, self.usedb, 'n')
h.store()
print >> sys.stderr, "Created new database in", self.dbname
def filter(self, msg):
h = hammie.open(self.dbname, self.usedb, 'r')
return h.filter(msg)
def filter_train(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
return h.filter(msg, train=True)
def train_ham(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.train_ham(msg, True)
h.store()
def train_spam(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.train_spam(msg, True)
h.store()
def untrain_ham(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.untrain_ham(msg)
h.store()
def untrain_spam(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.untrain_spam(msg)
h.store()
def main():
h = HammieFilter()
actions = []
opts, args = getopt.getopt(sys.argv[1:], 'hxd:D:nfgstGS',
['help', 'examples'])
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-x', '--examples'):
examples()
elif opt == '-d':
h.usedb = True
h.dbname = arg
elif opt == '-D':
h.usedb = False
h.dbname = arg
elif opt == '-f':
actions.append(h.filter)
elif opt == '-g':
actions.append(h.train_ham)
elif opt == '-s':
actions.append(h.train_spam)
elif opt == '-t':
actions.append(h.filter_train)
elif opt == '-G':
actions.append(h.untrain_ham)
elif opt == '-S':
actions.append(h.untrain_spam)
elif opt == "-n":
h.newdb()
sys.exit(0)
if actions == []:
actions = [h.filter]
msg = mboxutils.get_message(sys.stdin)
for action in actions:
action(msg)
sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None)))
if __name__ == "__main__":
main()
--- NEW FILE: sb-imapfilter.py ---
#!/usr/bin/env python
"""An IMAP filter. An IMAP message box is scanned and all non-scored
messages are scored and (where necessary) filtered.
The original filter design owed much to isbg by Roger Binns
(http://www.rogerbinns.com/isbg).
Usage:
imapfilter [options]
note: option values with spaces in them must be enclosed
in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-t : train contents of spam folder and ham folder
-c : classify inbox
-h : help
-v : verbose mode
-p : security option to prompt for imap password,
rather than look in options["imap", "password"]
-e y/n : expunge/purge messages on exit (y) or not (n)
-i debuglvl : a somewhat mysterious imaplib debugging level
-l minutes : period of time between filtering operations
-b : Launch a web browser showing the user interface.
Examples:
Classify inbox, with dbm database
imapfilter -c -D bayes.db
Train Spam and Ham, then classify inbox, with dbm database
imapfilter -t -c -D bayes.db
Train Spam and Ham only, with pickled database
imapfilter -t -d bayes.db
Warnings:
o This is alpha software! The filter is currently being developed and
tested. We do *not* recommend using it on a production system unless
you are confident that you can get your mail back if you lose it. On
the other hand, we do recommend that you test it for us and let us
know if anything does go wrong.
o By default, the filter does *not* delete, modify or move any of your
mail. Due to quirks in how imap works, new versions of your mail are
modified and placed in new folders, but the originals are still
available. These are flagged with the /Deleted flag so that you know
that they can be removed. Your mailer may not show these messages
by default, but there should be an option to do so. *However*, if
your mailer automatically purges/expunges (i.e. permanently deletes)
mail flagged as such, *or* if you set the imap_expunge option to
True, then this mail will be irretrievably lost.
To Do:
o IMAPMessage and IMAPFolder currently carry out very simple checks
of responses received from IMAP commands, but if the response is not
"OK", then the filter terminates. Handling of these errors could be
much nicer.
o IMAP over SSL is untested.
o Develop a test script, like spambayes/test/test_pop3proxy.py that
runs through some tests (perhaps with a *real* imap server, rather
than a dummy one). This would make it easier to carry out the tests
against each server whenever a change is made.
o IMAP supports authentication via other methods than the plain-text
password method that we are using at the moment. Neither of the
servers I have access to offer any alternative method, however. If
someone's does, then it would be nice to offer this.
o Usernames should be able to be literals as well as quoted strings.
This might help if the username/password has special characters like
accented characters.
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer , Tim Stone"
__credits__ = "All the Spambayes folk."
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import socket
import os
import re
import time
import sys
import getopt
import types
import email
import email.Parser
from getpass import getpass
from email.Utils import parsedate
from spambayes.Options import options
from spambayes import tokenizer, storage, message, Dibbler
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ImapUI import IMAPUserInterface
from spambayes.Version import get_version_string
from imaplib import IMAP4
from imaplib import Time2Internaldate
try:
if options["imap", "use_ssl"]:
from imaplib import IMAP_SSL as BaseIMAP
else:
from imaplib import IMAP4 as BaseIMAP
except ImportError:
from imaplib import IMAP4 as BaseIMAP
# global IMAPlib object
global imap
imap = None
# A flag can have any character in the ascii range 32-126
# except for (){ %*"\
FLAG_CHARS = ""
for i in range(32, 127):
if not chr(i) in ['(', ')', '{', ' ', '%', '*', '"', '\\']:
FLAG_CHARS += chr(i)
FLAG = r"\\?[" + re.escape(FLAG_CHARS) + r"]+"
# The empty flag set "()" doesn't match, so that extract returns
# data["FLAGS"] == None
FLAGS_RE = re.compile(r"(FLAGS) (\((" + FLAG + r" )*(" + FLAG + r")\))")
INTERNALDATE_RE = re.compile(r"(INTERNALDATE) (\"\d{1,2}\-[A-Za-z]{3,3}\-" +
r"\d{2,4} \d{2,2}\:\d{2,2}\:\d{2,2} " +
r"[\+\-]\d{4,4}\")")
RFC822_RE = re.compile(r"(RFC822) (\{[\d]+\})")
RFC822_HEADER_RE = re.compile(r"(RFC822.HEADER) (\{[\d]+\})")
UID_RE = re.compile(r"(UID) ([\d]+)")
FETCH_RESPONSE_RE = re.compile(r"([0-9]+) \(([" + \
re.escape(FLAG_CHARS) + r"\"\{\}\(\)\\ ]*)\)?")
LITERAL_RE = re.compile(r"^\{[\d]+\}$")
def _extract_fetch_data(response):
'''Extract data from the response given to an IMAP FETCH command.'''
# Response might be a tuple containing literal data
# At the moment, we only handle one literal per response. This
# may need to be improved if our code ever asks for something
# more complicated (like RFC822.Header and RFC822.Body)
if type(response) == types.TupleType:
literal = response[1]
response = response[0]
else:
literal = None
# the first item will always be the message number
mo = FETCH_RESPONSE_RE.match(response)
data = {}
if mo is None:
print """IMAP server gave strange fetch response. Please
report this as a bug."""
print response
else:
data["message_number"] = mo.group(1)
response = mo.group(2)
# We support the following FETCH items:
# FLAGS
# INTERNALDATE
# RFC822
# UID
# RFC822.HEADER
# All others are ignored.
for r in [FLAGS_RE, INTERNALDATE_RE, RFC822_RE, UID_RE,
RFC822_HEADER_RE]:
mo = r.search(response)
if mo is not None:
if LITERAL_RE.match(mo.group(2)):
data[mo.group(1)] = literal
else:
data[mo.group(1)] = mo.group(2)
return data
class IMAPSession(BaseIMAP):
'''A class extending the IMAP4 class, with a few optimizations'''
def __init__(self, server, port, debug=0, do_expunge=False):
try:
BaseIMAP.__init__(self, server, port)
except:
# A more specific except would be good here, but I get
# (in Python 2.2) a generic 'error' and a 'gaierror'
# if I pass a valid domain that isn't an IMAP server
# or invalid domain (respectively)
print "Invalid server or port, please check these settings."
sys.exit(-1)
self.debug = debug
# For efficiency, we remember which folder we are currently
# in, and only send a select command to the IMAP server if
# we want to *change* folders. This function is used by
# both IMAPMessage and IMAPFolder.
self.current_folder = None
self.do_expunge = do_expunge
def login(self, username, pwd):
try:
BaseIMAP.login(self, username, pwd) # superclass login
except BaseIMAP.error, e:
if str(e) == "permission denied":
print "There was an error logging in to the IMAP server."
print "The userid and/or password may be incorrect."
sys.exit()
else:
raise
def logout(self):
# sign off
if self.do_expunge:
self.expunge()
BaseIMAP.logout(self) # superclass logout
def SelectFolder(self, folder):
'''A method to point ensuing imap operations at a target folder'''
if self.current_folder != folder:
if self.current_folder != None:
if self.do_expunge:
# It is faster to do close() than a single
# expunge when we log out (because expunge returns
# a list of all the deleted messages, that we don't do
# anything with)
imap.close()
# We *always* use SELECT and not EXAMINE, because this
# speeds things up considerably.
response = self.select(folder, None)
if response[0] != "OK":
print "Invalid response to select %s:\n%s" % (folder,
response)
sys.exit(-1)
self.current_folder = folder
return response
def folder_list(self):
'''Return a alphabetical list of all folders available on the
server'''
response = self.list()
if response[0] != "OK":
return []
all_folders = response[1]
folders = []
for fol in all_folders:
# Sigh. Some servers may give us back the folder name as a
# literal, so we need to crunch this out.
if isinstance(fol, ()):
r = re.compile(r"{\d+}")
m = r.search(fol[0])
if not m:
# Something is wrong here! Skip this folder
continue
fol = '%s"%s"' % (fol[0][:m.start()], fol[1])
r = re.compile(r"\(([\w\\ ]*)\) ")
m = r.search(fol)
if not m:
# Something is not good with this folder, so skip it.
continue
name_attributes = fol[:m.end()-1]
# IMAP is a truly odd protocol. The delimiter is
# only the delimiter for this particular folder - each
# folder *may* have a different delimiter
self.folder_delimiter = fol[m.end()+1:m.end()+2]
# a bit of a hack, but we really need to know if this is
# the case
if self.folder_delimiter == ',':
print """WARNING: Your imap server uses commas as the folder
delimiter. This may cause unpredictable errors."""
folders.append(fol[m.end()+5:-1])
folders.sort()
return folders
def FindMessage(self, id):
'''A (potentially very expensive) method to find a message with
a given spambayes id (header), and return a message object (no
substance).'''
# If efficiency becomes a concern, what we could do is store a
# dict of key-to-folder, and look in that folder first. (It might
# have moved independantly of us, so we would still have to search
# if we didn't find it). For the moment, we do an search through
# all folders, alphabetically.
for folder_name in self.folder_list():
fol = IMAPFolder(folder_name)
for msg in fol:
if msg.id == id:
return msg
return None
class IMAPMessage(message.SBHeaderMessage):
def __init__(self):
message.Message.__init__(self)
self.folder = None
self.previous_folder = None
self.rfc822_command = "RFC822.PEEK"
self.got_substance = False
def setFolder(self, folder):
self.folder = folder
def _check(self, response, command):
if response[0] != "OK":
print "Invalid response to %s:\n%s" % (command, response)
sys.exit(-1)
def extractTime(self):
# When we create a new copy of a message, we need to specify
# a timestamp for the message. If the message has a valid date
# header we use that. Otherwise, we use the current time.
message_date = self["Date"]
if message_date is not None:
parsed_date = parsedate(message_date)
if parsed_date is not None:
return Time2Internaldate(time.mktime(parsed_date))
else:
return Time2Internaldate(time.time())
def get_substance(self):
'''Retrieve the RFC822 message from the IMAP server and set as the
substance of this message.'''
if self.got_substance:
return
if not self.uid or not self.id:
print "Cannot get substance of message without an id and an UID"
return
imap.SelectFolder(self.folder.name)
# We really want to use RFC822.PEEK here, as that doesn't effect
# the status of the message. Unfortunately, it appears that not
# all IMAP servers support this, even though it is in RFC1730
# Actually, it's not: we should be using BODY.PEEK
try:
response = imap.uid("FETCH", self.uid, self.rfc822_command)
except IMAP4.error:
self.rfc822_command = "RFC822"
response = imap.uid("FETCH", self.uid, self.rfc822_command)
if response[0] != "OK":
self.rfc822_command = "RFC822"
response = imap.uid("FETCH", self.uid, self.rfc822_command)
self._check(response, "uid fetch")
data = _extract_fetch_data(response[1][0])
# Annoyingly, we can't just pass over the RFC822 message to an
# existing message object (like self) and have it parse it. So
# we go through the hoops of creating a new message, and then
# copying over all its internals.
new_msg = email.Parser.Parser().parsestr(data["RFC822"])
self._headers = new_msg._headers
self._unixfrom = new_msg._unixfrom
self._payload = new_msg._payload
self._charset = new_msg._charset
self.preamble = new_msg.preamble
self.epilogue = new_msg.epilogue
self._default_type = new_msg._default_type
if not self.has_key(options["Headers", "mailid_header_name"]):
self[options["Headers", "mailid_header_name"]] = self.id
self.got_substance = True
if options["globals", "verbose"]:
sys.stdout.write(chr(8) + "*")
def MoveTo(self, dest):
'''Note that message should move to another folder. No move is
carried out until Save() is called, for efficiency.'''
if self.previous_folder is None:
self.previous_folder = self.folder
self.folder = dest
def Save(self):
'''Save message to imap server.'''
# we can't actually update the message with IMAP
# so what we do is create a new message and delete the old one
if self.folder is None:
raise RuntimeError, """Can't save a message that doesn't
have a folder."""
if not self.id:
raise RuntimeError, """Can't save a message that doesn't have
an id."""
response = imap.uid("FETCH", self.uid, "(FLAGS INTERNALDATE)")
self._check(response, 'fetch (flags internaldate)')
data = _extract_fetch_data(response[1][0])
if data.has_key("INTERNALDATE"):
msg_time = data["INTERNALDATE"]
else:
msg_time = self.extractTime()
if data.has_key("FLAGS"):
flags = data["FLAGS"]
# The \Recent flag can be fetched, but cannot be stored
# We must remove it from the list if it is there.
flags = re.sub(r"\\Recent ?|\\ ?Recent", "", flags)
else:
flags = None
response = imap.append(self.folder.name, flags,
msg_time, self.as_string())
if response[0] == "NO":
# This may be because we have tried to set an invalid flag.
# Try again, losing all the flag information, but warn the
# user that this has happened.
response = imap.append(self.folder.name, None, msg_time,
self.as_string())
if response[0] == "OK":
print "WARNING: Could not append flags: %s" % (flags,)
self._check(response, 'append')
if self.previous_folder is None:
imap.SelectFolder(self.folder.name)
else:
imap.SelectFolder(self.previous_folder.name)
self.previous_folder = None
response = imap.uid("STORE", self.uid, "+FLAGS.SILENT", "(\\Deleted)")
self._check(response, 'store')
# We need to update the uid, as it will have changed.
# Although we don't use the UID to keep track of messages, we do
# have to use it for IMAP operations.
imap.SelectFolder(self.folder.name)
response = imap.uid("SEARCH", "(UNDELETED HEADER " + \
options["Headers", "mailid_header_name"] + \
" " + self.id + ")")
self._check(response, 'search')
new_id = response[1][0]
# Let's hope it doesn't, but, just in case, if the search
# turns up empty, we make the assumption that the new
# message is the last one with a recent flag
if new_id == "":
response = imap.uid("SEARCH", "RECENT")
new_id = response[1][0]
if new_id.find(' ') > -1:
ids = new_id.split(' ')
new_id = ids[-1]
# Ok, now we're in trouble if we still haven't found it.
# We make a huge assumption that the new message is the one
# with the highest UID (they are sequential, so this will be
# ok as long as another message hasn't also arrived)
if new_id == "":
response = imap.uid("SEARCH", "ALL")
new_id = response[1][0]
if new_id.find(' ') > -1:
ids = new_id.split(' ')
new_id = ids[-1]
self.uid = new_id
# This performs a similar function to email.message_from_string()
def imapmessage_from_string(s, _class=IMAPMessage, strict=False):
return email.message_from_string(s, _class, strict)
class IMAPFolder(object):
def __init__(self, folder_name):
self.name = folder_name
# Unique names for cached messages - see _generate_id below.
self.lastBaseMessageName = ''
self.uniquifier = 2
def __cmp__(self, obj):
'''Two folders are equal if their names are equal'''
if obj is None:
return False
return cmp(self.name, obj.name)
def _check(self, response, command):
if response[0] != "OK":
print "Invalid response to %s:\n%s" % (command, response)
sys.exit(-1)
def __iter__(self):
'''IMAPFolder is iterable'''
for key in self.keys():
try:
yield self[key]
except KeyError:
pass
def recent_uids(self):
'''Returns uids for all the messages in the folder that
are flagged as recent, but not flagged as deleted.'''
imap.SelectFolder(self.name, True)
response = imap.uid("SEARCH", "RECENT UNDELETED")
self._check(response, "SEARCH RECENT UNDELETED")
return response[1][0].split(' ')
def keys(self):
'''Returns *uids* for all the messages in the folder not
marked as deleted.'''
imap.SelectFolder(self.name)
response = imap.uid("SEARCH", "UNDELETED")
self._check(response, "SEARCH UNDELETED")
if response[1][0] == "":
return []
return response[1][0].split(' ')
def __getitem__(self, key):
'''Return message (no substance) matching the given *uid*.'''
# We don't retrieve the substances of the message here - you need
# to call msg.get_substance() to do that.
imap.SelectFolder(self.name)
# Using RFC822.HEADER.LINES would be better here, but it seems
# that not all servers accept it, even though it is in the RFC
response = imap.uid("FETCH", key, "RFC822.HEADER")
self._check(response, "uid fetch header")
data = _extract_fetch_data(response[1][0])
msg = IMAPMessage()
msg.setFolder(self)
msg.uid = key
r = re.compile(re.escape(options["Headers",
"mailid_header_name"]) + \
"\:\s*(\d+(\-\d)?)")
mo = r.search(data["RFC822.HEADER"])
if mo is None:
msg.setId(self._generate_id())
# Unfortunately, we now have to re-save this message, so that
# our id is stored on the IMAP server. Before anyone suggests
# it, we can't store it as a flag, because user-defined flags
# aren't supported by all IMAP servers.
# This will need to be done once per message.
msg.get_substance()
msg.Save()
else:
msg.setId(mo.group(1))
if options["globals", "verbose"]:
sys.stdout.write(".")
return msg
# Lifted straight from pop3proxy.py (under the name getNewMessageName)
def _generate_id(self):
# The message id is the time it arrived, with a uniquifier
# appended if two arrive within one clock tick of each other.
messageName = "%10.10d" % long(time.time())
if messageName == self.lastBaseMessageName:
messageName = "%s-%d" % (messageName, self.uniquifier)
self.uniquifier += 1
else:
self.lastBaseMessageName = messageName
self.uniquifier = 2
return messageName
def Train(self, classifier, isSpam):
'''Train folder as spam/ham'''
num_trained = 0
for msg in self:
if msg.GetTrained() == (not isSpam):
msg.get_substance()
msg.delSBHeaders()
classifier.unlearn(msg.asTokens(), not isSpam)
# Once the message has been untrained, it's training memory
# should reflect that on the off chance that for some reason
# the training breaks, which happens on occasion (the
# tokenizer is not yet perfect)
msg.RememberTrained(None)
if msg.GetTrained() is None:
msg.get_substance()
msg.delSBHeaders()
classifier.learn(msg.asTokens(), isSpam)
num_trained += 1
msg.RememberTrained(isSpam)
if isSpam:
move_opt_name = "move_trained_spam_to_folder"
else:
move_opt_name = "move_trained_ham_to_folder"
if options["imap", move_opt_name] != "":
msg.MoveTo(IMAPFolder(options["imap",
move_opt_name]))
msg.Save()
return num_trained
def Filter(self, classifier, spamfolder, unsurefolder):
count = {}
count["ham"] = 0
count["spam"] = 0
count["unsure"] = 0
for msg in self:
if msg.GetClassification() is None:
msg.get_substance()
(prob, clues) = classifier.spamprob(msg.asTokens(),
evidence=True)
# add headers and remember classification
msg.addSBHeaders(prob, clues)
cls = msg.GetClassification()
if cls == options["Hammie", "header_ham_string"]:
# we leave ham alone
count["ham"] += 1
elif cls == options["Hammie", "header_spam_string"]:
msg.MoveTo(spamfolder)
count["spam"] += 1
else:
msg.MoveTo(unsurefolder)
count["unsure"] += 1
msg.Save()
return count
class IMAPFilter(object):
def __init__(self, classifier):
self.spam_folder = IMAPFolder(options["imap", "spam_folder"])
self.unsure_folder = IMAPFolder(options["imap", "unsure_folder"])
self.classifier = classifier
def Train(self):
if options["globals", "verbose"]:
t = time.time()
total_ham_trained = 0
total_spam_trained = 0
if options["imap", "ham_train_folders"] != "":
ham_training_folders = options["imap", "ham_train_folders"]
for fol in ham_training_folders:
# Select the folder to make sure it exists
imap.SelectFolder(fol)
if options['globals', 'verbose']:
print " Training ham folder %s" % (fol)
folder = IMAPFolder(fol)
num_ham_trained = folder.Train(self.classifier, False)
total_ham_trained += num_ham_trained
if options['globals', 'verbose']:
print " %s trained." % (num_ham_trained)
if options["imap", "spam_train_folders"] != "":
spam_training_folders = options["imap", "spam_train_folders"]
for fol in spam_training_folders:
# Select the folder to make sure it exists
imap.SelectFolder(fol)
if options['globals', 'verbose']:
print " Training spam folder %s" % (fol)
folder = IMAPFolder(fol)
num_spam_trained = folder.Train(self.classifier, True)
total_spam_trained += num_spam_trained
if options['globals', 'verbose']:
print " %s trained." % (num_spam_trained)
if total_ham_trained or total_spam_trained:
self.classifier.store()
if options["globals", "verbose"]:
print "Training took %s seconds, %s messages were trained" \
% (time.time() - t, total_ham_trained + total_spam_trained)
def Filter(self):
if options["globals", "verbose"]:
t = time.time()
count = None
# Select the spam folder and unsure folder to make sure they exist
imap.SelectFolder(self.spam_folder.name)
imap.SelectFolder(self.unsure_folder.name)
for filter_folder in options["imap", "filter_folders"]:
# Select the folder to make sure it exists
imap.SelectFolder(filter_folder)
folder = IMAPFolder(filter_folder)
count = folder.Filter(self.classifier, self.spam_folder,
self.unsure_folder)
if options["globals", "verbose"]:
if count is not None:
print "\nClassified %s ham, %s spam, and %s unsure." % \
(count["ham"], count["spam"], count["unsure"])
print "Classifying took", time.time() - t, "seconds."
def run():
global imap
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbtcvpl:e:i:d:D:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = options["Storage", "persistent_storage_file"]
useDBM = options["Storage", "persistent_use_database"]
doTrain = False
doClassify = False
doExpunge = options["imap", "expunge"]
imapDebug = 0
sleepTime = 0
promptForPass = False
launchUI = False
server = ""
username = ""
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == "-b":
launchUI = True
elif opt == '-t':
doTrain = True
elif opt == '-p':
promptForPass = True
elif opt == '-c':
doClassify = True
elif opt == '-v':
options["globals", "verbose"] = True
elif opt == '-e':
if arg == 'y':
doExpunge = True
else:
doExpunge = False
elif opt == '-i':
imapDebug = int(arg)
elif opt == '-l':
sleepTime = int(arg) * 60
# Let the user know what they are using...
print get_version_string("IMAP Filter")
print "and engine %s.\n" % (get_version_string(),)
if not (doClassify or doTrain or launchUI):
print "-b, -c, or -t operands must be specified."
print "Please use the -h operand for help."
sys.exit()
if (launchUI and (doClassify or doTrain)):
print """
-b option is exclusive with -c and -t options.
The user interface will be launched, but no classification
or training will be performed."""
bdbname = os.path.expanduser(bdbname)
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
classifier = storage.open_storage(bdbname, useDBM)
if options["globals", "verbose"]:
print "Done."
if options["imap", "server"]:
# The options class is ahead of us here:
# it knows that imap:server will eventually be able to have
# multiple values, but for the moment, we just use the first one
server = options["imap", "server"]
if len(server) > 0:
server = server[0]
username = options["imap", "username"]
if len(username) > 0:
username = username[0]
if not promptForPass:
pwd = options["imap", "password"]
if len(pwd) > 0:
pwd = pwd[0]
else:
pwd = None
if not launchUI:
print "You need to specify both a server and a username."
sys.exit()
if promptForPass:
pwd = getpass()
if server.find(':') > -1:
server, port = server.split(':', 1)
port = int(port)
else:
if options["imap", "use_ssl"]:
port = 993
else:
port = 143
imap_filter = IMAPFilter(classifier)
# Web interface
if launchUI:
if server != "":
imap = IMAPSession(server, port, imapDebug, doExpunge)
httpServer = UserInterfaceServer(options["html_ui", "port"])
httpServer.register(IMAPUserInterface(classifier, imap, pwd))
Dibbler.run(launchBrowser=launchUI)
else:
while True:
imap = IMAPSession(server, port, imapDebug, doExpunge)
imap.login(username, pwd)
if doTrain:
if options["globals", "verbose"]:
print "Training"
imap_filter.Train()
if doClassify:
if options["globals", "verbose"]:
print "Classifying"
imap_filter.Filter()
imap.logout()
if sleepTime:
time.sleep(sleepTime)
else:
break
if __name__ == '__main__':
run()
--- NEW FILE: sb-mailsort.py ---
#! /usr/bin/env python
"""\
To train:
%(program)s -t ham.mbox spam.mbox
To filter mail (using .forward or .qmail):
|%(program)s Maildir/ Mail/Spam/
To print the score and top evidence for a message or messages:
%(program)s -s message [message ...]
"""
SPAM_CUTOFF = 0.57
SIZE_LIMIT = 5000000 # messages larger are not analyzed
BLOCK_SIZE = 10000
RC_DIR = "~/.spambayes"
DB_FILE = RC_DIR + "/wordprobs.cdb"
CONFIG_FILE = RC_DIR + "/bayescustomize.ini"
import sys
import os
import getopt
import email
import time
import signal
import socket
import email
DB_FILE = os.path.expanduser(DB_FILE)
def import_spambayes():
global mboxutils, CdbClassifier, tokenize
if not os.environ.has_key('BAYESCUSTOMIZE'):
os.environ['BAYESCUSTOMIZE'] = os.path.expanduser(CONFIG_FILE)
from spambayes import mboxutils
from spambayes.cdb_classifier import CdbClassifier
from spambayes.tokenizer import tokenize
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
program = sys.argv[0] # For usage(); referenced by docstring above
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def maketmp(dir):
hostname = socket.gethostname()
pid = os.getpid()
fd = -1
for x in xrange(200):
filename = "%d.%d.%s" % (time.time(), pid, hostname)
pathname = "%s/tmp/%s" % (dir, filename)
try:
fd = os.open(pathname, os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0600)
except IOError, exc:
if exc[i] not in (errno.EINT, errno.EEXIST):
raise
else:
break
time.sleep(2)
if fd == -1:
raise SystemExit, "could not create a mail file"
return (os.fdopen(fd, "wb"), pathname, filename)
def train(bayes, msgs, is_spam):
"""Train bayes with all messages from a mailbox."""
mbox = mboxutils.getmbox(msgs)
for msg in mbox:
bayes.learn(tokenize(msg), is_spam)
def train_messages(ham_name, spam_name):
"""Create database using messages."""
rc_dir = os.path.expanduser(RC_DIR)
if not os.path.exists(rc_dir):
print "Creating", RC_DIR, "directory..."
os.mkdir(rc_dir)
bayes = CdbClassifier()
print 'Training with ham...'
train(bayes, ham_name, False)
print 'Training with spam...'
train(bayes, spam_name, True)
print 'Update probabilities and writing DB...'
db = open(DB_FILE, "wb")
bayes.save_wordinfo(db)
db.close()
print 'done'
def filter_message(hamdir, spamdir):
signal.signal(signal.SIGALRM, lambda s: sys.exit(1))
signal.alarm(24 * 60 * 60)
# write message to temporary file (must be on same partition)
tmpfile, pathname, filename = maketmp(hamdir)
try:
tmpfile.write(os.environ.get("DTLINE", "")) # delivered-to line
bytes = 0
blocks = []
while 1:
block = sys.stdin.read(BLOCK_SIZE)
if not block:
break
bytes += len(block)
if bytes < SIZE_LIMIT:
blocks.append(block)
tmpfile.write(block)
tmpfile.close()
if bytes < SIZE_LIMIT:
msgdata = ''.join(blocks)
del blocks
msg = email.message_from_string(msgdata)
del msgdata
bayes = CdbClassifier(open(DB_FILE, 'rb'))
prob = bayes.spamprob(tokenize(msg))
else:
prob = 0.0
if prob > SPAM_CUTOFF:
os.rename(pathname, "%s/new/%s" % (spamdir, filename))
else:
os.rename(pathname, "%s/new/%s" % (hamdir, filename))
except:
os.unlink(pathname)
raise
def print_message_score(msg_name, msg_fp):
msg = email.message_from_file(msg_fp)
bayes = CdbClassifier(open(DB_FILE, 'rb'))
prob, evidence = bayes.spamprob(tokenize(msg), evidence=True)
print msg_name, prob
for word, prob in evidence:
print ' ', `word`, prob
def main():
global DB_FILE, CONFIG_FILE
try:
opts, args = getopt.getopt(sys.argv[1:], 'tsd:c:')
except getopt.error, msg:
usage(2, msg)
mode = 'sort'
for opt, val in opts:
if opt == '-t':
mode = 'train'
elif opt == '-s':
mode = 'score'
elif opt == '-d':
DB_FILE = val
elif opt == '-c':
CONFIG_FILE = val
else:
assert 0, 'invalid option'
import_spambayes()
if mode == 'sort':
if len(args) != 2:
usage(2, 'wrong number of arguments')
filter_message(args[0], args[1])
elif mode == 'train':
if len(args) != 2:
usage(2, 'wrong number of arguments')
train_messages(args[0], args[1])
elif mode == 'score':
if args:
for msg in args:
print_message_score(msg, open(msg))
else:
print_message_score('', sys.stdin)
if __name__ == "__main__":
main()
--- NEW FILE: sb-mboxtrain.py ---
#! /usr/bin/env python
### Train spambayes on all previously-untrained messages in a mailbox.
###
### This keeps track of messages it's already trained by adding an
### X-Spambayes-Trained: header to each one. Then, if you move one to
### another folder, it will retrain that message. You would want to run
### this from a cron job on your server.
"""Usage: %(program)s [OPTIONS] ...
Where OPTIONS is one or more of:
-h
show usage and exit
-d DBNAME
use the DBM store. A DBM file is larger than the pickle and
creating it is slower, but loading it is much faster,
especially for large word databases. Recommended for use with
hammiefilter or any procmail-based filter.
-D DBNAME
use the pickle store. A pickle is smaller and faster to create,
but much slower to load. Recommended for use with pop3proxy and
hammiesrv.
-g PATH
mbox or directory of known good messages (non-spam) to train on.
Can be specified more than once.
-s PATH
mbox or directory of known spam messages to train on.
Can be specified more than once.
-f
force training, ignoring the trained header. Use this if you
need to rebuild your database from scratch.
-q
quiet mode; no output
-n train mail residing in "new" directory, in addition to "cur" directory,
which is always trained (Maildir only)
-r remove mail which was trained on (Maildir only)
"""
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import sys, os, getopt
from spambayes import hammie, mboxutils
from spambayes.Options import options
program = sys.argv[0]
TRAINED_HDR = "X-Spambayes-Trained"
loud = True
def msg_train(h, msg, is_spam, force):
"""Train bayes with a single message."""
# XXX: big hack -- why is email.Message unable to represent
# multipart/alternative?
try:
msg.as_string()
except TypeError:
# We'll be unable to represent this as text :(
return False
if is_spam:
spamtxt = options["Headers", "header_spam_string"]
else:
spamtxt = options["Headers", "header_ham_string"]
oldtxt = msg.get(TRAINED_HDR)
if force:
# Train no matter what.
if oldtxt != None:
del msg[TRAINED_HDR]
elif oldtxt == spamtxt:
# Skip this one, we've already trained with it.
return False
elif oldtxt != None:
# It's been trained, but as something else. Untrain.
del msg[TRAINED_HDR]
h.untrain(msg, not is_spam)
h.train(msg, is_spam)
msg.add_header(TRAINED_HDR, spamtxt)
return True
def maildir_train(h, path, is_spam, force, removetrained):
"""Train bayes with all messages from a maildir."""
if loud: print " Reading %s as Maildir" % (path,)
import time
import socket
pid = os.getpid()
host = socket.gethostname()
counter = 0
trained = 0
for fn in os.listdir(path):
cfn = os.path.join(path, fn)
tfn = os.path.normpath(os.path.join(path, "..", "tmp",
"%d.%d_%d.%s" % (time.time(), pid,
counter, host)))
if (os.path.isdir(cfn)):
continue
counter += 1
if loud:
sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(cfn, "rb")
msg = mboxutils.get_message(f)
f.close()
if not msg_train(h, msg, is_spam, force):
continue
trained += 1
f = file(tfn, "wb")
f.write(msg.as_string())
f.close()
# XXX: This will raise an exception on Windows. Do any Windows
# people actually use Maildirs?
os.rename(tfn, cfn)
if (removetrained):
os.unlink(cfn)
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def mbox_train(h, path, is_spam, force):
"""Train bayes with a Unix mbox"""
if loud: print " Reading as Unix mbox"
import mailbox
import fcntl
import tempfile
# Open and lock the mailbox. Some systems require it be opened for
# writes in order to assert an exclusive lock.
f = file(path, "r+b")
fcntl.flock(f, fcntl.LOCK_EX)
mbox = mailbox.PortableUnixMailbox(f, mboxutils.get_message)
outf = os.tmpfile()
counter = 0
trained = 0
for msg in mbox:
counter += 1
if loud:
sys.stdout.write(" %s\r" % counter)
sys.stdout.flush()
if msg_train(h, msg, is_spam, force):
trained += 1
# Write it out with the Unix "From " line
outf.write(msg.as_string(True))
outf.seek(0)
try:
os.ftruncate(f.fileno(), 0)
f.seek(0)
except:
# If anything goes wrong, don't try to write
print "Problem truncating mbox--nothing written"
raise
try:
for line in outf.xreadlines():
f.write(line)
except:
print >> sys.stderr ("Problem writing mbox! Sorry, "
"I tried my best, but your mail "
"may be corrupted.")
raise
fcntl.lockf(f, fcntl.LOCK_UN)
f.close()
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def mhdir_train(h, path, is_spam, force):
"""Train bayes with an mh directory"""
if loud: print " Reading as MH mailbox"
import glob
counter = 0
trained = 0
for fn in glob.glob(os.path.join(path, "[0-9]*")):
counter += 1
cfn = fn
tfn = os.path.join(path, "spambayes.tmp")
if loud:
sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(fn, "rb")
msg = mboxutils.get_message(f)
f.close()
msg_train(h, msg, is_spam, force)
trained += 1
f = file(tfn, "wb")
f.write(msg.as_string())
f.close()
# XXX: This will raise an exception on Windows. Do any Windows
# people actually use MH directories?
os.rename(tfn, cfn)
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def train(h, path, is_spam, force, trainnew, removetrained):
if not os.path.exists(path):
raise ValueError("Nonexistent path: %s" % path)
elif os.path.isfile(path):
mbox_train(h, path, is_spam, force)
elif os.path.isdir(os.path.join(path, "cur")):
maildir_train(h, os.path.join(path, "cur"), is_spam, force, removetrained)
if trainnew:
maildir_train(h, os.path.join(path, "new"), is_spam, force, removetrained)
elif os.path.isdir(path):
mhdir_train(h, path, is_spam, force)
else:
raise ValueError("Unable to determine mailbox type: " + path)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def main():
"""Main program; parse options and go."""
global loud
try:
opts, args = getopt.getopt(sys.argv[1:], 'hfqnrd:D:g:s:')
except getopt.error, msg:
usage(2, msg)
if not opts:
usage(2, "No options given")
pck = None
usedb = None
force = False
trainnew = False
removetrained = False
good = []
spam = []
for opt, arg in opts:
if opt == '-h':
usage(0)
elif opt == "-f":
force = True
elif opt == "-n":
trainnew = True
elif opt == "-q":
loud = False
elif opt == '-g':
good.append(arg)
elif opt == '-s':
spam.append(arg)
elif opt == "-r":
removetrained = True
elif opt == "-d":
usedb = True
pck = arg
elif opt == "-D":
usedb = False
pck = arg
if args:
usage(2, "Positional arguments not allowed")
if usedb == None:
usage(2, "Must specify one of -d or -D")
h = hammie.open(pck, usedb, "c")
for g in good:
if loud: print "Training ham (%s):" % g
train(h, g, False, force, trainnew, removetrained)
save = True
for s in spam:
if loud: print "Training spam (%s):" % s
train(h, s, True, force, trainnew, removetrained)
save = True
if save:
h.store()
if __name__ == "__main__":
main()
--- NEW FILE: sb-notesfilter.py ---
#! /usr/bin/env python
'''notesfilter.py - Lotus Notes Spambayes interface.
Classes:
Abstract:
This module uses Spambayes as a filter against a Lotus Notes mail
database. The Notes client must be running when this process is
executed.
It requires a Notes folder, named as a parameter, with four
subfolders:
Spam
Ham
Train as Spam
Train as Ham
Depending on the execution parameters, it will do any or all of the
following steps, in the order given.
1. Train Spam from the Train as Spam folder (-t option)
2. Train Ham from the Train as Ham folder (-t option)
3. Replicate (-r option)
4. Classify the inbox (-c option)
Mail that is to be trained as spam should be manually moved to
that folder by the user. Likewise mail that is to be trained as
ham. After training, spam is moved to the Spam folder and ham is
moved to the Ham folder.
Replication takes place if a remote server has been specified.
This step may take a long time, depending on replication
parameters and how much information there is to download, as well
as line speed and server load. Please be patient if you run with
replication. There is currently no progress bar or anything like
that to tell you that it's working, but it is and will complete
eventually. There is also no mechanism for notifying you that the
replication failed. If it did, there is no harm done, and the program
will continue execution.
Mail that is classified as Spam is moved from the inbox to the
Train as Spam folder. You should occasionally review your Spam
folder for Ham that has mistakenly been classified as Spam. If
there is any there, move it to the Train as Ham folder, so
Spambayes will be less likely to make this mistake again.
Mail that is classified as Ham or Unsure is left in the inbox.
There is currently no means of telling if a mail was classified as
Ham or Unsure.
You should occasionally select some Ham and move it to the Train
as Ham folder, so Spambayes can tell the difference between Spam
and Ham. The goal is to maintain a relative balance between the
number of Spam and the number of Ham that have been trained into
the database. These numbers are reported every time this program
executes. However, if the amount of Spam you receive far exceeds
the amount of Ham you receive, it may be very difficult to
maintain this balance. This is not a matter of great concern.
Spambayes will still make very few mistakes in this circumstance.
But, if this is the case, you should review your Spam folder for
falsely classified Ham, and retrain those that you find, on a
regular basis. This will prevent statistical error accumulation,
which if allowed to continue, would cause Spambayes to tend to
classify everything as Spam.
Because there is no programmatic way to determine if a particular
mail has been previously processed by this classification program,
it keeps a pickled dictionary of notes mail ids, so that once a
mail has been classified, it will not be classified again. The
non-existence of is index file, named .sbindex,
indicates to the system that this is an initialization execution.
Rather than classify the inbox in this case, the contents of the
inbox are placed in the index to note the 'starting point' of the
system. After that, any new messages in the inbox are eligible
for classification.
Usage:
notesfilter [options]
note: option values with spaces in them must be enclosed
in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-l dbname : database filename of local mail replica
e.g. localmail.nsf
-r server : server address of the server mail database
e.g. d27ml602/27/M/IBM
if specified, will initiate a replication
-f folder : Name of spambayes folder
must have subfolders: Spam
Ham
Train as Spam
Train as Ham
-t : train contents of Train as Spam and Train as Ham
-c : classify inbox
-h : help
-p : prompt "Press Enter to end" before ending
This is useful for automated executions where the
statistics output would otherwise be lost when the
window closes.
Examples:
Replicate and classify inbox
notesfilter -c -d notesbayes -r mynoteserv -l mail.nsf -f Spambayes
Train Spam and Ham, then classify inbox
notesfilter -t -c -d notesbayes -l mail.nsf -f Spambayes
Replicate, then classify inbox
notesfilter -c -d test7 -l mail.nsf -r nynoteserv -f Spambayes
To Do:
o Dump/purge notesindex file
o Create correct folders if they do not exist
o Options for some of this stuff?
o pop3proxy style training/configuration interface?
o parameter to retrain?
o Suggestions?
'''
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tim Stone "
__credits__ = "Mark Hammond, for his remarkable win32 modules."
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
def bool(val):
return not not val
import sys
from spambayes import tokenizer, storage
from spambayes.Options import options
import cPickle as pickle
import errno
import win32com.client
import pywintypes
import getopt
def classifyInbox(v, vmoveto, bayes, ldbname, notesindex):
# the notesindex hash ensures that a message is looked at only once
if len(notesindex.keys()) == 0:
firsttime = 1
else:
firsttime = 0
docstomove = []
numham = 0
numspam = 0
numuns = 0
numdocs = 0
doc = v.GetFirstDocument()
while doc:
nid = doc.NOTEID
if firsttime:
notesindex[nid] = 'never classified'
else:
if not notesindex.has_key(nid):
numdocs += 1
# Notes returns strings in unicode, and the Python
# uni-decoder has trouble with these strings when
# you try to print them. So don't...
# The com interface returns basic data types as tuples
# only, thus the subscript on GetItemValue
try:
subj = doc.GetItemValue('Subject')[0]
except:
subj = 'No Subject'
try:
body = doc.GetItemValue('Body')[0]
except:
body = 'No Body'
message = "Subject: %s\r\n\r\n%s" % (subj, body)
# generate_long_skips = True blows up on occasion,
# probably due to this unicode problem.
options["Tokenizer", "generate_long_skips"] = False
tokens = tokenizer.tokenize(message)
prob, clues = bayes.spamprob(tokens, evidence=True)
if prob < options["Categorization", "ham_cutoff"]:
disposition = options["Hammie", "header_ham_string"]
numham += 1
elif prob > options["Categorization", "spam_cutoff"]:
disposition = options["Hammie", "header_spam_string"]
docstomove += [doc]
numspam += 1
else:
disposition = options["Hammie", "header_unsure_string"]
numuns += 1
notesindex[nid] = 'classified'
try:
print "%s spamprob is %s" % (subj[:30], prob)
except UnicodeError:
print " spamprob is %s" % (prob)
doc = v.GetNextDocument(doc)
# docstomove list is built because moving documents in the middle of
# the classification loop looses the iterator position
for doc in docstomove:
doc.RemoveFromFolder(v.Name)
doc.PutInFolder(vmoveto.Name)
print "%s documents processed" % (numdocs)
print " %s classified as spam" % (numspam)
print " %s classified as ham" % (numham)
print " %s classified as unsure" % (numuns)
def processAndTrain(v, vmoveto, bayes, is_spam, notesindex):
if is_spam:
str = options["Hammie", "header_spam_string"]
else:
str = options["Hammie", "header_ham_string"]
print "Training %s" % (str)
docstomove = []
doc = v.GetFirstDocument()
while doc:
try:
subj = doc.GetItemValue('Subject')[0]
except:
subj = 'No Subject'
try:
body = doc.GetItemValue('Body')[0]
except:
body = 'No Body'
message = "Subject: %s\r\n%s" % (subj, body)
options["Tokenizer", "generate_long_skips"] = False
tokens = tokenizer.tokenize(message)
nid = doc.NOTEID
if notesindex.has_key(nid):
trainedas = notesindex[nid]
if trainedas == options["Hammie", "header_spam_string"] and \
not is_spam:
# msg is trained as spam, is to be retrained as ham
bayes.unlearn(tokens, True)
elif trainedas == options["Hammie", "header_ham_string"] and \
is_spam:
# msg is trained as ham, is to be retrained as spam
bayes.unlearn(tokens, False)
bayes.learn(tokens, is_spam)
notesindex[nid] = str
docstomove += [doc]
doc = v.GetNextDocument(doc)
for doc in docstomove:
doc.RemoveFromFolder(v.Name)
doc.PutInFolder(vmoveto.Name)
print "%s documents trained" % (len(docstomove))
def run(bdbname, useDBM, ldbname, rdbname, foldname, doTrain, doClassify):
if useDBM:
bayes = storage.DBDictClassifier(bdbname)
else:
bayes = storage.PickledClassifier(bdbname)
try:
fp = open("%s.sbindex" % (ldbname), 'rb')
except IOError, e:
if e.errno != errno.ENOENT: raise
notesindex = {}
print "%s.sbindex file not found, this is a first time run" \
% (ldbname)
print "No classification will be performed"
else:
notesindex = pickle.load(fp)
fp.close()
sess = win32com.client.Dispatch("Lotus.NotesSession")
try:
sess.initialize()
except pywintypes.com_error:
print "Session aborted"
sys.exit()
db = sess.GetDatabase("",ldbname)
vinbox = db.getView('($Inbox)')
vspam = db.getView("%s\Spam" % (foldname))
vham = db.getView("%s\Ham" % (foldname))
vtrainspam = db.getView("%s\Train as Spam" % (foldname))
vtrainham = db.getView("%s\Train as Ham" % (foldname))
if doTrain:
processAndTrain(vtrainspam, vspam, bayes, True, notesindex)
# for some reason, using inbox as a target here loses the mail
processAndTrain(vtrainham, vham, bayes, False, notesindex)
if rdbname:
print "Replicating..."
db.Replicate(rdbname)
print "Done"
if doClassify:
classifyInbox(vinbox, vtrainspam, bayes, ldbname, notesindex)
print "The Spambayes database currently has %s Spam and %s Ham" \
% (bayes.nspam, bayes.nham)
bayes.store()
fp = open("%s.sbindex" % (ldbname), 'wb')
pickle.dump(notesindex, fp)
fp.close()
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], 'htcpd:D:l:r:f:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = None # bayes database name
ldbname = None # local notes database name
rdbname = None # remote notes database location
sbfname = None # spambayes folder name
doTrain = False
doClassify = False
doPrompt = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == '-l':
ldbname = arg
elif opt == '-r':
rdbname = arg
elif opt == '-f':
sbfname = arg
elif opt == '-t':
doTrain = True
elif opt == '-c':
doClassify = True
elif opt == '-p':
doPrompt = True
if (bdbname and ldbname and sbfname and (doTrain or doClassify)):
run(bdbname, useDBM, ldbname, rdbname, \
sbfname, doTrain, doClassify)
if doPrompt:
try:
key = input("Press Enter to end")
except SyntaxError:
pass
else:
print >>sys.stderr, __doc__
--- NEW FILE: sb-pop3dnd.py ---
#!/usr/bin/env python
from __future__ import generators
"""
Overkill (someone *please* come up with something to call this script!)
This application is a twisted cross between a POP3 proxy and an IMAP
server. It sits between your mail client and your POP3 server (like any
other POP3 proxy). While messages classified as ham are simply passed
through the proxy, messages that are classified as spam or unsure are
intercepted and passed to the IMAP server. The IMAP server offers three
folders - one where messages classified as spam end up, one for messages
it is unsure about, and one for training ham.
In other words, to use this application, setup your mail client to connect
to localhost, rather than directly to your POP3 server. Additionally, add
a new IMAP account, also connecting to localhost. Setup the application
via the web interface, and you are ready to go. Good messages will appear
as per normal, but you will also have two new incoming folders, one for
spam and one for ham.
To train SpamBayes, use the spam folder, and the 'train_as_ham' folder.
Any messages in these folders will be trained appropriately. This means
that all messages that SpamBayes classifies as spam will also be trained
as such. If you receive any 'false positives' (ham classified as spam),
you *must* copy the message into the 'train_as_ham' folder to correct the
training. You may also place any saved spam messages you have into this
folder.
So that SpamBayes knows about ham as well as spam, you will also need to
move or copy mail into the 'train_as_ham' folder. These may come from
the unsure folder, or from any other mail you have saved. It is a good
idea to leave messages in the 'train_as_ham' and 'spam' folders, so that
you can retrain from scratch if required. (However, you should always
clear out your unsure folder, preferably moving or copying the messages
into the appropriate training folder).
This SpamBayes application is designed to work with Outlook Express, and
provide the same sort of ease of use as the Outlook plugin. Although the
majority of development and testing has been done with Outlook Express,
any mail client that supports both IMAP and POP3 should be able to use this
application - if the client enables the user to work with an IMAP account
and POP3 account side-by-side (and move messages between them), then it
should work equally as well as Outlook Express.
This module includes the following classes:
o IMAPFileMessage
o IMAPFileMessageFactory
o IMAPMailbox
o SpambayesMailbox
o Trainer
o SpambayesAccount
o SpambayesIMAPServer
o OneParameterFactory
o MyBayesProxy
o MyBayesProxyListener
o IMAPState
"""
todo = """
o Message flags are currently not persisted, but should be. The
IMAPFileMessage class should be extended to do this. The same
goes for the 'internaldate' of the message.
o The RECENT flag should be unset at some point, but when? The
RFC says that a message is recent if this is the first session
to be notified about the message. Perhaps this can be done
simply by *not* persisting this flag - i.e. the flag is always
loaded as not recent, and only new messages are recent. The
RFC says that if it is not possible to determine, then all
messages should be recent, and this is what we currently do.
o The Mailbox should be calling the appropriate listener
functions (currently only newMessages is called on addMessage).
flagsChanged should also be called on store, addMessage, or ???
o We cannot currently get part of a message via the BODY calls
(with the <> operands), or get a part of a MIME message (by
prepending a number). This should be added!
o If the user clicks the 'save and shutdown' button on the web
interface, this will only kill the POP3 proxy and web interface
threads, and not the IMAP server. We need to monitor the thread
that we kick off, and if it dies, we should die too. Need to figure
out how to do this in twisted.
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "All the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import os
import re
import sys
import md5
import time
import errno
import types
import thread
import getopt
import imaplib
import operator
import StringIO
import email.Utils
from twisted import cred
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.app import Application
from twisted.internet.defer import maybeDeferred
from twisted.internet.protocol import ServerFactory
from twisted.protocols.imap4 import parseNestedParens, parseIdList
from twisted.protocols.imap4 import IllegalClientResponse, IAccount
from twisted.protocols.imap4 import collapseNestedLists, MessageSet
from twisted.protocols.imap4 import IMAP4Server, MemoryAccount, IMailbox
from twisted.protocols.imap4 import IMailboxListener, collapseNestedLists
# Provide for those that don't have spambayes on their PYTHONPATH
sys.path.insert(-1, os.path.dirname(os.getcwd()))
from spambayes.Options import options
from spambayes.message import Message
from spambayes.tokenizer import tokenize
from spambayes import FileCorpus, Dibbler
from spambayes.Version import get_version_string
from spambayes.ServerUI import ServerUserInterface
from spambayes.UserInterface import UserInterfaceServer
from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState
def ensureDir(dirname):
"""Ensure that the given directory exists - in other words, if it
does not exist, attempt to create it."""
try:
os.mkdir(dirname)
if options["globals", "verbose"]:
print "Creating directory", dirname
except OSError, e:
if e.errno != errno.EEXIST:
raise
class IMAPFileMessage(FileCorpus.FileMessage):
'''IMAP Message that persists as a file system artifact.'''
def __init__(self, file_name, directory):
"""Constructor(message file name, corpus directory name)."""
FileCorpus.FileMessage.__init__(self, file_name, directory)
self.id = file_name
self.directory = directory
self.date = imaplib.Time2Internaldate(time.time())[1:-1]
self.clear_flags()
# IMessage implementation
def getHeaders(self, negate, names):
"""Retrieve a group of message headers."""
headers = {}
if not isinstance(names, tuple):
names = (names,)
for header, value in self.items():
if (header.upper() in names and not negate) or names == ():
headers[header.upper()] = value
return headers
def getFlags(self):
"""Retrieve the flags associated with this message."""
return self._flags_iter()
def _flags_iter(self):
if self.deleted:
yield "\\DELETED"
if self.answered:
yield "\\ANSWERED"
if self.flagged:
yield "\\FLAGGED"
if self.seen:
yield "\\SEEN"
if self.draft:
yield "\\DRAFT"
if self.draft:
yield "\\RECENT"
def getInternalDate(self):
"""Retrieve the date internally associated with this message."""
return self.date
def getBodyFile(self):
"""Retrieve a file object containing the body of this message."""
# Note only body, not headers!
s = StringIO.StringIO()
s.write(self.body())
s.seek(0)
return s
#return file(os.path.join(self.directory, self.id), "r")
def getSize(self):
"""Retrieve the total size, in octets, of this message."""
return len(self.as_string())
def getUID(self):
"""Retrieve the unique identifier associated with this message."""
return self.id
def getSubPart(self, part):
"""Retrieve a MIME sub-message
@type part: C{int}
@param part: The number of the part to retrieve, indexed from 0.
@rtype: Any object implementing C{IMessage}.
@return: The specified sub-part.
"""
# IMessage implementation ends
def clear_flags(self):
"""Set all message flags to false."""
self.deleted = False
self.answered = False
self.flagged = False
self.seen = False
self.draft = False
self.recent = False
def set_flag(self, flag, value):
# invalid flags are ignored
flag = flag.upper()
if flag == "\\DELETED":
self.deleted = value
elif flag == "\\ANSWERED":
self.answered = value
elif flag == "\\FLAGGED":
self.flagged = value
elif flag == "\\SEEN":
self.seen = value
elif flag == "\\DRAFT":
self.draft = value
else:
print "Tried to set invalid flag", flag, "to", value
def flags(self):
"""Return the message flags."""
all_flags = []
if self.deleted:
all_flags.append("\\DELETED")
if self.answered:
all_flags.append("\\ANSWERED")
if self.flagged:
all_flags.append("\\FLAGGED")
if self.seen:
all_flags.append("\\SEEN")
if self.draft:
all_flags.append("\\DRAFT")
if self.draft:
all_flags.append("\\RECENT")
return all_flags
def train(self, classifier, isSpam):
if self.GetTrained() == (not isSpam):
classifier.unlearn(self.asTokens(), not isSpam)
self.RememberTrained(None)
if self.GetTrained() is None:
classifier.learn(self.asTokens(), isSpam)
self.RememberTrained(isSpam)
classifier.store()
def structure(self, ext=False):
"""Body structure data describes the MIME-IMB
format of a message and consists of a sequence of mime type, mime
subtype, parameters, content id, description, encoding, and size.
The fields following the size field are variable: if the mime
type/subtype is message/rfc822, the contained message's envelope
information, body structure data, and number of lines of text; if
the mime type is text, the number of lines of text. Extension fields
may also be included; if present, they are: the MD5 hash of the body,
body disposition, body language."""
s = []
for part in self.walk():
if part.get_content_charset() is not None:
charset = ("charset", part.get_content_charset())
else:
charset = None
part_s = [part.get_main_type(), part.get_subtype(),
charset,
part.get('Content-Id'),
part.get('Content-Description'),
part.get('Content-Transfer-Encoding'),
str(len(part.as_string()))]
#if part.get_type() == "message/rfc822":
# part_s.extend([envelope, body_structure_data,
# part.as_string().count("\n")])
#elif part.get_main_type() == "text":
if part.get_main_type() == "text":
part_s.append(str(part.as_string().count("\n")))
if ext:
part_s.extend([md5.new(part.as_string()).digest(),
part.get('Content-Disposition'),
part.get('Content-Language')])
s.append(part_s)
if len(s) == 1:
return s[0]
return s
def body(self):
rfc822 = self.as_string()
bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
re.DOTALL + re.MULTILINE)
bmatch = bodyRE.search(rfc822)
return bmatch.group(2)
def headers(self):
rfc822 = self.as_string()
bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
re.DOTALL + re.MULTILINE)
bmatch = bodyRE.search(rfc822)
return rfc822[:bmatch.start(2)]
def on(self, date1, date2):
"contained within the date"
raise NotImplementedError
def before(self, date1, date2):
"before the date"
raise NotImplementedError
def since(self, date1, date2):
"within or after the date"
raise NotImplementedError
def string_contains(self, whole, sub):
return whole.find(sub) != -1
def matches(self, criteria):
"""Return True iff the messages matches the specified IMAP
criteria."""
match_tests = {"ALL" : [(True, True)],
"ANSWERED" : [(self.answered, True)],
"DELETED" : [(self.deleted, True)],
"DRAFT" : [(self.draft, True)],
"FLAGGED" : [(self.flagged, True)],
"NEW" : [(self.recent, True), (self.seen, False)],
"RECENT" : [(self.recent, True)],
"SEEN" : [(self.seen, True)],
"UNANSWERED" : [(self.answered, False)],
"UNDELETED" : [(self.deleted, False)],
"UNDRAFT" : [(self.draft, False)],
"UNFLAGGED" : [(self.flagged, False)],
"UNSEEN" : [(self.seen, False)],
"OLD" : [(self.recent, False)],
}
complex_tests = {"BCC" : (self.string_contains, self.get("Bcc")),
"SUBJECT" : (self.string_contains, self.get("Subject")),
"CC" : (self.string_contains, self.get("Cc")),
"BODY" : (self.string_contains, self.body()),
"TO" : (self.string_contains, self.get("To")),
"TEXT" : (self.string_contains, self.as_string()),
"FROM" : (self.string_contains, self.get("From")),
"SMALLER" : (operator.lt, len(self.as_string())),
"LARGER" : (operator.gt, len(self.as_string())),
"BEFORE" : (self.before, self.date),
"ON" : (self.on, self.date),
"SENTBEFORE" : (self.before, self.get("Date")),
"SENTON" : (self.on, self.get("Date")),
"SENTSINCE" : (self.since, self.get("Date")),
"SINCE" : (self.since, self.date),
}
result = True
test = None
header = None
header_field = None
for c in criteria:
if match_tests.has_key(c) and test is None and header is None:
for (test, result) in match_tests[c]:
result = result and (test == result)
elif complex_tests.has_key(c) and test is None and header is None:
test = complex_tests[c]
elif test is not None and header is None:
result = result and test[0](test[1], c)
test = None
elif c == "HEADER" and test is None:
# the only criteria that uses the next _two_ elements
header = c
elif test is None and header is not None and header_field is None:
header_field = c
elif header is not None and header_field is not None and test is None:
result = result and self.string_contains(self.get(header_field), c)
header = None
header_field = None
return result
"""
Still to do:
Messages with message sequence numbers
corresponding to the specified message sequence
number set
UID
Messages with unique identifiers corresponding to
the specified unique identifier set.
KEYWORD Messages with the specified keyword set.
UNKEYWORD
Messages that do not have the specified keyword
set.
NOT
Messages that do not match the specified search
key.
OR
Messages that match either search key.
"""
class IMAPFileMessageFactory(FileCorpus.FileMessageFactory):
'''MessageFactory for IMAPFileMessage objects'''
def create(self, key, directory):
'''Create a message object from a filename in a directory'''
return IMAPFileMessage(key, directory)
class IMAPMailbox(cred.perspective.Perspective):
__implements__ = (IMailbox,)
def __init__(self, name, identity_name, id):
cred.perspective.Perspective.__init__(self, name, identity_name)
self.UID_validity = id
self.listeners = []
def getUIDValidity(self):
"""Return the unique validity identifier for this mailbox."""
return self.UID_validity
def addListener(self, listener):
"""Add a mailbox change listener."""
self.listeners.append(listener)
def removeListener(self, listener):
"""Remove a mailbox change listener."""
self.listeners.remove(listener)
class SpambayesMailbox(IMAPMailbox):
def __init__(self, name, id, directory):
IMAPMailbox.__init__(self, name, "spambayes", id)
self.UID_validity = id
ensureDir(directory)
self.storage = FileCorpus.FileCorpus(IMAPFileMessageFactory(),
directory, r"[0123456789]*")
# UIDs are required to be strictly ascending.
if len(self.storage.keys()) == 0:
self.nextUID = 0
else:
self.nextUID = long(self.storage.keys()[-1]) + 1
# Calculate initial recent and unseen counts
# XXX Note that this will always end up with zero counts
# XXX until the flags are persisted.
self.unseen_count = 0
self.recent_count = 0
for msg in self.storage:
if not msg.seen:
self.unseen_count += 1
if msg.recent:
self.recent_count += 1
def getUIDNext(self, increase=False):
"""Return the likely UID for the next message added to this
mailbox."""
reply = str(self.nextUID)
if increase:
self.nextUID += 1
return reply
def getUID(self, message):
"""Return the UID of a message in the mailbox."""
# Note that IMAP messages are 1-based, our messages are 0-based
d = self.storage
return long(d.keys()[message - 1])
def getFlags(self):
"""Return the flags defined in this mailbox."""
return ["\\Answered", "\\Flagged", "\\Deleted", "\\Seen",
"\\Draft"]
def getMessageCount(self):
"""Return the number of messages in this mailbox."""
return len(self.storage.keys())
def getRecentCount(self):
"""Return the number of messages with the 'Recent' flag."""
return self.recent_count
def getUnseenCount(self):
"""Return the number of messages with the 'Unseen' flag."""
return self.unseen_count
def isWriteable(self):
"""Get the read/write status of the mailbox."""
return True
def destroy(self):
"""Called before this mailbox is deleted, permanently."""
# Our mailboxes cannot be deleted
raise NotImplementedError
def getHierarchicalDelimiter(self):
"""Get the character which delimits namespaces for in this
mailbox."""
return '.'
def requestStatus(self, names):
"""Return status information about this mailbox."""
answer = {}
for request in names:
request = request.upper()
if request == "MESSAGES":
answer[request] = self.getMessageCount()
elif request == "RECENT":
answer[request] = self.getRecentCount()
elif request == "UIDNEXT":
answer[request] = self.getUIDNext()
elif request == "UIDVALIDITY":
answer[request] = self.getUIDValidity()
elif request == "UNSEEN":
answer[request] = self.getUnseenCount()
return answer
def addMessage(self, message, flags=(), date=None):
"""Add the given message to this mailbox."""
msg = self.storage.makeMessage(self.getUIDNext(True))
msg.date = date
msg.setPayload(message.read())
self.storage.addMessage(msg)
self.store(MessageSet(long(msg.id), long(msg.id)), flags, 1, True)
msg.recent = True
msg.store()
self.recent_count += 1
self.unseen_count += 1
for listener in self.listeners:
listener.newMessages(self.getMessageCount(),
self.getRecentCount())
d = defer.Deferred()
reactor.callLater(0, d.callback, self.storage.keys().index(msg.id))
return d
def expunge(self):
"""Remove all messages flagged \\Deleted."""
deleted_messages = []
for msg in self.storage:
if msg.deleted:
if not msg.seen:
self.unseen_count -= 1
if msg.recent:
self.recent_count -= 1
deleted_messages.append(long(msg.id))
self.storage.removeMessage(msg)
if deleted_messages != []:
for listener in self.listeners:
listener.newMessages(self.getMessageCount(),
self.getRecentCount())
return deleted_messages
def search(self, query, uid):
"""Search for messages that meet the given query criteria.
@type query: C{list}
@param query: The search criteria
@rtype: C{list}
@return: A list of message sequence numbers or message UIDs which
match the search criteria.
"""
if self.getMessageCount() == 0:
return []
all_msgs = MessageSet(long(self.storage.keys()[0]),
long(self.storage.keys()[-1]))
matches = []
for id, msg in self._messagesIter(all_msgs, uid):
for q in query:
if msg.matches(q):
matches.append(id)
break
return matches
def _messagesIter(self, messages, uid):
if uid:
messages.last = long(self.storage.keys()[-1])
else:
messages.last = self.getMessageCount()
for id in messages:
if uid:
msg = self.storage.get(str(id))
else:
msg = self.storage.get(str(self.getUID(id)))
if msg is None:
# Non-existant message.
continue
msg.load()
yield (id, msg)
def fetch(self, messages, uid):
"""Retrieve one or more messages."""
return self._messagesIter(messages, uid)
def store(self, messages, flags, mode, uid):
"""Set the flags of one or more messages."""
stored_messages = {}
for id, msg in self._messagesIter(messages, uid):
if mode == 0:
msg.clear_flags()
value = True
elif mode == -1:
value = False
elif mode == 1:
value = True
for flag in flags:
if flag == '(' or flag == ')':
continue
if flag == "SEEN" and value == True and msg.seen == False:
self.unseen_count -= 1
if flag == "SEEN" and value == False and msg.seen == True:
self.unseen_count += 1
msg.set_flag(flag, value)
stored_messages[id] = msg.flags()
return stored_messages
class Trainer(object):
"""Listens to a given mailbox and trains new messages as spam or
ham."""
__implements__ = (IMailboxListener,)
def __init__(self, mailbox, asSpam):
self.mailbox = mailbox
self.asSpam = asSpam
def modeChanged(self, writeable):
# We don't care
pass
def flagsChanged(self, newFlags):
# We don't care
pass
def newMessages(self, exists, recent):
# We don't get passed the actual message, or the id of
# the message, of even the message number. We just get
# the total number of new/recent messages.
# However, this function should be called _every_ time
# that a new message appears, so we should be able to
# assume that the last message is the new one.
# (We ignore the recent count)
if exists is not None:
id = self.mailbox.getUID(exists)
msg = self.mailbox.storage[str(id)]
msg.train(state.bayes, self.asSpam)
class SpambayesAccount(MemoryAccount):
"""Account for Spambayes server."""
def __init__(self, id, ham, spam, unsure):
MemoryAccount.__init__(self, id)
self.mailboxes = {"SPAM" : spam,
"UNSURE" : unsure,
"TRAIN_AS_HAM" : ham}
def select(self, name, readwrite=1):
# 'INBOX' is a special case-insensitive name meaning the
# primary mailbox for the user; we interpret this as an alias
# for 'spam'
if name.upper() == "INBOX":
name = "SPAM"
return MemoryAccount.select(self, name, readwrite)
class SpambayesIMAPServer(IMAP4Server):
IDENT = "Spambayes IMAP Server IMAP4rev1 Ready"
def __init__(self, user_account):
IMAP4Server.__init__(self)
self.account = user_account
def authenticateLogin(self, user, passwd):
"""Lookup the account associated with the given parameters."""
if user == options["imapserver", "username"] and \
passwd == options["imapserver", "password"]:
return (IAccount, self.account, None)
raise cred.error.UnauthorizedLogin()
def connectionMade(self):
state.activeIMAPSessions += 1
state.totalIMAPSessions += 1
IMAP4Server.connectionMade(self)
def connectionLost(self, reason):
state.activeIMAPSessions -= 1
IMAP4Server.connectionLost(self, reason)
def do_CREATE(self, tag, args):
"""Creating new folders on the server is not permitted."""
self.sendNegativeResponse(tag, \
"Creation of new folders is not permitted")
auth_CREATE = (do_CREATE, IMAP4Server.arg_astring)
select_CREATE = auth_CREATE
def do_DELETE(self, tag, args):
"""Deleting folders on the server is not permitted."""
self.sendNegativeResponse(tag, \
"Deletion of folders is not permitted")
auth_DELETE = (do_DELETE, IMAP4Server.arg_astring)
select_DELETE = auth_DELETE
class OneParameterFactory(ServerFactory):
"""A factory that allows a single parameter to be passed to the created
protocol."""
def buildProtocol(self, addr):
"""Create an instance of a subclass of Protocol, passing a single
parameter."""
if self.parameter is not None:
p = self.protocol(self.parameter)
else:
p = self.protocol()
p.factory = self
return p
class MyBayesProxy(POP3ProxyBase):
"""Proxies between an email client and a POP3 server, redirecting
mail to the imap server as necessary. It acts on the following
POP3 commands:
o RETR:
o Adds the judgement header based on the raw headers and body
of the message.
"""
intercept_message = 'From: "Spambayes" \n' \
'Subject: Spambayes Intercept\n\nA message ' \
'was intercepted by Spambayes (it scored %s).\n' \
'\nYou may find it in the Spam or Unsure ' \
'folder.\n\n.\n'
def __init__(self, clientSocket, serverName, serverPort, spam, unsure):
POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'RETR': self.onRetr}
state.totalSessions += 1
state.activeSessions += 1
self.isClosed = False
self.spam_folder = spam
self.unsure_folder = unsure
def send(self, data):
"""Logs the data to the log file."""
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
try:
return POP3ProxyBase.send(self, data)
except socket.error:
self.close()
def recv(self, size):
"""Logs the data to the log file."""
data = POP3ProxyBase.recv(self, size)
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
return data
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
state.activeSessions -= 1
POP3ProxyBase.close(self)
def onTransaction(self, command, args, response):
"""Takes the raw request and response, and returns the
(possibly processed) response to pass back to the email client.
"""
handler = self.handlers.get(command, self.onUnknown)
return handler(command, args, response)
def onRetr(self, command, args, response):
"""Classifies the message. If the result is ham, then simply
pass it through. If the result is an unsure or spam, move it
to the appropriate IMAP folder."""
# Use '\n\r?\n' to detect the end of the headers in case of
# broken emails that don't use the proper line separators.
if re.search(r'\n\r?\n', response):
# Break off the first line, which will be '+OK'.
ok, messageText = response.split('\n', 1)
prob = state.bayes.spamprob(tokenize(messageText))
if prob < options["Categorization", "ham_cutoff"]:
# Return the +OK and the message with the header added.
state.numHams += 1
return ok + "\n" + messageText
elif prob > options["Categorization", "spam_cutoff"]:
dest_folder = self.spam_folder
state.numSpams += 1
else:
dest_folder = self.unsure_folder
state.numUnsure += 1
msg = StringIO.StringIO(messageText)
date = imaplib.Time2Internaldate(time.time())[1:-1]
dest_folder.addMessage(msg, (), date)
# We have to return something, because the client is expecting
# us to. We return a short message indicating that a message
# was intercepted.
return ok + "\n" + self.intercept_message % (prob,)
else:
# Must be an error response.
return response
def onUnknown(self, command, args, response):
"""Default handler; returns the server's response verbatim."""
return response
class MyBayesProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
MyBayesProxy objects to serve them.
"""
def __init__(self, serverName, serverPort, proxyPort, spam, unsure):
proxyArgs = (serverName, serverPort, spam, unsure)
Dibbler.Listener.__init__(self, proxyPort, MyBayesProxy, proxyArgs)
print 'Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class IMAPState(State):
def __init__(self):
State.__init__(self)
# Set up the extra statistics.
self.totalIMAPSessions = 0
self.activeIMAPSessions = 0
def buildServerStrings(self):
"""After the server details have been set up, this creates string
versions of the details, for display in the Status panel."""
self.serverPortString = str(self.imap_port)
# Also build proxy strings
State.buildServerStrings(self)
state = IMAPState()
# ===================================================================
# __main__ driver.
# ===================================================================
def setup():
# Setup app, boxes, trainers and account
proxyListeners = []
app = Application("SpambayesIMAPServer")
spam_box = SpambayesMailbox("Spam", 0, options["imapserver",
"spam_directory"])
unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver",
"unsure_directory"])
ham_train_box = SpambayesMailbox("TrainAsHam", 2,
options["imapserver", "ham_directory"])
spam_trainer = Trainer(spam_box, True)
ham_trainer = Trainer(ham_train_box, False)
spam_box.addListener(spam_trainer)
ham_train_box.addListener(ham_trainer)
user_account = SpambayesAccount(options["imapserver", "username"],
ham_train_box, spam_box, unsure_box)
# add IMAP4 server
f = OneParameterFactory()
f.protocol = SpambayesIMAPServer
f.parameter = user_account
state.imap_port = options["imapserver", "port"]
app.listenTCP(state.imap_port, f)
# add POP3 proxy
state.createWorkers()
for (server, serverPort), proxyPort in zip(state.servers,
state.proxyPorts):
listener = MyBayesProxyListener(server, serverPort, proxyPort,
spam_box, unsure_box)
proxyListeners.append(listener)
state.buildServerStrings()
# add web interface
httpServer = UserInterfaceServer(state.uiPort)
serverUI = ServerUserInterface(state, _recreateState)
httpServer.register(serverUI)
return app
def run():
# Read the arguments.
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbd:D:u:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
launchUI = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-b':
launchUI = True
elif opt == '-d': # dbm file
state.useDB = True
options["Storage", "persistent_storage_file"] = arg
elif opt == '-D': # pickle file
state.useDB = False
options["Storage", "persistent_storage_file"] = arg
elif opt == '-u':
state.uiPort = int(arg)
# Let the user know what they are using...
print get_version_string("IMAP Server")
print "and engine %s.\n" % (get_version_string(),)
# setup everything
app = setup()
# kick things off
thread.start_new_thread(Dibbler.run, (launchUI,))
app.run(save=False)
if __name__ == "__main__":
run()
--- NEW FILE: sb-server.py ---
#!/usr/bin/env python
"""A POP3 proxy that works with classifier.py, and adds a simple
X-Spambayes-Classification header (ham/spam/unsure) to each incoming
email. You point pop3proxy at your POP3 server, and configure your
email client to collect mail from the proxy then filter on the added
header. Usage:
pop3proxy.py [options] [ []]
is the name of your real POP3 server
is the port number of your real POP3 server, which
defaults to 110.
options:
-h : Displays this help message.
-d FILE : use the named DBM database file
-D FILE : the the named Pickle database file
-l port : proxy listens on this port number (default 110)
-u port : User interface listens on this port number
(default 8880; Browse http://localhost:8880/)
-b : Launch a web browser showing the user interface.
All command line arguments and switches take their default
values from the [pop3proxy] and [html_ui] sections of
bayescustomize.ini.
For safety, and to help debugging, the whole POP3 conversation is
written out to _pop3proxy.log for each run, if
options["globals", "verbose"] is True.
To make rebuilding the database easier, uploaded messages are appended
to _pop3proxyham.mbox and _pop3proxyspam.mbox.
"""
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Richie Hindle "
__credits__ = "Tim Peters, Neale Pickett, Tim Stone, all the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
todo = """
Web training interface:
User interface improvements:
o Once the pieces are on separate pages, make the paste box bigger.
o Deployment: Windows executable? atlaxwin and ctypes? Or just
webbrowser?
o Save the stats (num classified, etc.) between sessions.
o "Reload database" button.
New features:
o Online manual.
o Links to project homepage, mailing list, etc.
o List of words with stats (it would have to be paged!) a la SpamSieve.
Code quality:
o Cope with the email client timing out and closing the connection.
Info:
o Slightly-wordy index page; intro paragraph for each page.
o In both stats and training results, report nham and nspam - warn if
they're very different (for some value of 'very').
o "Links" section (on homepage?) to project homepage, mailing list,
etc.
Gimmicks:
o Classify a web page given a URL.
o Graphs. Of something. Who cares what?
o NNTP proxy.
o Zoe...!
"""
import os, sys, re, errno, getopt, time, traceback, socket, cStringIO
from thread import start_new_thread
from email.Header import Header
import spambayes.message
from spambayes import Dibbler
from spambayes import storage
from spambayes.FileCorpus import FileCorpus, ExpiryFileCorpus
from spambayes.FileCorpus import FileMessageFactory, GzipFileMessageFactory
from spambayes.Options import options
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ProxyUI import ProxyUserInterface
from spambayes.Version import get_version_string
# Increase the stack size on MacOS X. Stolen from Lib/test/regrtest.py
if sys.platform == 'darwin':
try:
import resource
except ImportError:
pass
else:
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
newsoft = min(hard, max(soft, 1024*2048))
resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))
# number to add to STAT length for each msg to fudge for spambayes headers
HEADER_SIZE_FUDGE_FACTOR = 512
class ServerLineReader(Dibbler.BrighterAsyncChat):
"""An async socket that reads lines from a remote server and
simply calls a callback with the data. The BayesProxy object
can't connect to the real POP3 server and talk to it
synchronously, because that would block the process."""
lineCallback = None
def __init__(self, serverName, serverPort, lineCallback):
Dibbler.BrighterAsyncChat.__init__(self)
self.lineCallback = lineCallback
self.request = ''
self.set_terminator('\r\n')
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((serverName, serverPort))
except socket.error, e:
error = "Can't connect to %s:%d: %s" % (serverName, serverPort, e)
print >>sys.stderr, error
self.lineCallback('-ERR %s\r\n' % error)
self.lineCallback('') # "The socket's been closed."
self.close()
def collect_incoming_data(self, data):
self.request = self.request + data
def found_terminator(self):
self.lineCallback(self.request + '\r\n')
self.request = ''
def handle_close(self):
self.lineCallback('')
self.close()
class POP3ProxyBase(Dibbler.BrighterAsyncChat):
"""An async dispatcher that understands POP3 and proxies to a POP3
server, calling `self.onTransaction(request, response)` for each
transaction. Responses are not un-byte-stuffed before reaching
self.onTransaction() (they probably should be for a totally generic
POP3ProxyBase class, but BayesProxy doesn't need it and it would
mean re-stuffing them afterwards). self.onTransaction() should
return the response to pass back to the email client - the response
can be the verbatim response or a processed version of it. The
special command 'KILL' kills it (passing a 'QUIT' command to the
server).
"""
def __init__(self, clientSocket, serverName, serverPort):
Dibbler.BrighterAsyncChat.__init__(self, clientSocket)
self.request = ''
self.response = ''
self.set_terminator('\r\n')
self.command = '' # The POP3 command being processed...
self.args = [] # ...and its arguments
self.isClosing = False # Has the server closed the socket?
self.seenAllHeaders = False # For the current RETR or TOP
self.startTime = 0 # (ditto)
self.serverSocket = ServerLineReader(serverName, serverPort,
self.onServerLine)
def onTransaction(self, command, args, response):
"""Overide this. Takes the raw request and the response, and
returns the (possibly processed) response to pass back to the
email client.
"""
raise NotImplementedError
def onServerLine(self, line):
"""A line of response has been received from the POP3 server."""
isFirstLine = not self.response
self.response = self.response + line
# Is this the line that terminates a set of headers?
self.seenAllHeaders = self.seenAllHeaders or line in ['\r\n', '\n']
# Has the server closed its end of the socket?
if not line:
self.isClosing = True
# If we're not processing a command, just echo the response.
if not self.command:
self.push(self.response)
self.response = ''
# Time out after 30 seconds for message-retrieval commands if
# all the headers are down. The rest of the message will proxy
# straight through.
if self.command in ['TOP', 'RETR'] and \
self.seenAllHeaders and time.time() > self.startTime + 30:
self.onResponse()
self.response = ''
# If that's a complete response, handle it.
elif not self.isMultiline() or line == '.\r\n' or \
(isFirstLine and line.startswith('-ERR')):
self.onResponse()
self.response = ''
def isMultiline(self):
"""Returns True if the request should get a multiline
response (assuming the response is positive).
"""
if self.command in ['USER', 'PASS', 'APOP', 'QUIT',
'STAT', 'DELE', 'NOOP', 'RSET', 'KILL']:
return False
elif self.command in ['RETR', 'TOP', 'CAPA']:
return True
elif self.command in ['LIST', 'UIDL']:
return len(self.args) == 0
else:
# Assume that an unknown command will get a single-line
# response. This should work for errors and for POP-AUTH,
# and is harmless even for multiline responses - the first
# line will be passed to onTransaction and ignored, then the
# rest will be proxied straight through.
return False
def collect_incoming_data(self, data):
"""Asynchat override."""
self.request = self.request + data
def found_terminator(self):
"""Asynchat override."""
verb = self.request.strip().upper()
if verb == 'KILL':
self.socket.shutdown(2)
self.close()
raise SystemExit
elif verb == 'CRASH':
# For testing
x = 0
y = 1/x
self.serverSocket.push(self.request + '\r\n')
if self.request.strip() == '':
# Someone just hit the Enter key.
self.command = ''
self.args = []
else:
# A proper command.
splitCommand = self.request.strip().split()
self.command = splitCommand[0].upper()
self.args = splitCommand[1:]
self.startTime = time.time()
self.request = ''
def onResponse(self):
# We don't support pipelining, so if the command is CAPA and the
# response includes PIPELINING, hack out that line of the response.
if self.command == 'CAPA':
pipelineRE = r'(?im)^PIPELINING[^\n]*\n'
self.response = re.sub(pipelineRE, '', self.response)
# Pass the request and the raw response to the subclass and
# send back the cooked response.
if self.response:
cooked = self.onTransaction(self.command, self.args, self.response)
self.push(cooked)
# If onServerLine() decided that the server has closed its
# socket, close this one when the response has been sent.
if self.isClosing:
self.close_when_done()
# Reset.
self.command = ''
self.args = []
self.isClosing = False
self.seenAllHeaders = False
class BayesProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
BayesProxy objects to serve them.
"""
def __init__(self, serverName, serverPort, proxyPort):
proxyArgs = (serverName, serverPort)
Dibbler.Listener.__init__(self, proxyPort, BayesProxy, proxyArgs)
print 'Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class BayesProxy(POP3ProxyBase):
"""Proxies between an email client and a POP3 server, inserting
judgement headers. It acts on the following POP3 commands:
o STAT:
o Adds the size of all the judgement headers to the maildrop
size.
o LIST:
o With no message number: adds the size of an judgement header
to the message size for each message in the scan listing.
o With a message number: adds the size of an judgement header
to the message size.
o RETR:
o Adds the judgement header based on the raw headers and body
of the message.
o TOP:
o Adds the judgement header based on the raw headers and as
much of the body as the TOP command retrieves. This can
mean that the header might have a different value for
different calls to TOP, or for calls to TOP vs. calls to
RETR. I'm assuming that the email client will either not
make multiple calls, or will cope with the headers being
different.
o USER:
o Does no processing based on the USER command itself, but
expires any old messages in the three caches.
"""
def __init__(self, clientSocket, serverName, serverPort):
POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'STAT': self.onStat, 'LIST': self.onList,
'RETR': self.onRetr, 'TOP': self.onTop,
'USER': self.onUser}
state.totalSessions += 1
state.activeSessions += 1
self.isClosed = False
def send(self, data):
"""Logs the data to the log file."""
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
try:
return POP3ProxyBase.send(self, data)
except socket.error:
# The email client has closed the connection - 40tude Dialog
# does this immediately after issuing a QUIT command,
# without waiting for the response.
self.close()
def recv(self, size):
"""Logs the data to the log file."""
data = POP3ProxyBase.recv(self, size)
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
return data
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
state.activeSessions -= 1
POP3ProxyBase.close(self)
def onTransaction(self, command, args, response):
"""Takes the raw request and response, and returns the
(possibly processed) response to pass back to the email client.
"""
handler = self.handlers.get(command, self.onUnknown)
return handler(command, args, response)
def onStat(self, command, args, response):
"""Adds the size of all the judgement headers to the maildrop
size."""
match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response)
if match:
count = int(match.group(1))
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR * count
return '+OK %d %d%s\r\n' % (count, size, match.group(3))
else:
return response
def onList(self, command, args, response):
"""Adds the size of an judgement header to the message
size(s)."""
if response.count('\r\n') > 1:
# Multiline: all lines but the first contain a message size.
lines = response.split('\r\n')
outputLines = [lines[0]]
for line in lines[1:]:
match = re.search(r'^(\d+)\s+(\d+)', line)
if match:
number = int(match.group(1))
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR
line = "%d %d" % (number, size)
outputLines.append(line)
return '\r\n'.join(outputLines)
else:
# Single line.
match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response)
if match:
messageNumber = match.group(1)
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR
trailer = match.group(3)
return "+OK %s %s%s\r\n" % (messageNumber, size, trailer)
else:
return response
def onRetr(self, command, args, response):
"""Adds the judgement header based on the raw headers and body
of the message."""
# Use '\n\r?\n' to detect the end of the headers in case of
# broken emails that don't use the proper line separators.
if re.search(r'\n\r?\n', response):
# Remove the trailing .\r\n before passing to the email parser.
# Thanks to Scott Schlesier for this fix.
terminatingDotPresent = (response[-4:] == '\n.\r\n')
if terminatingDotPresent:
response = response[:-3]
# Break off the first line, which will be '+OK'.
ok, messageText = response.split('\n', 1)
try:
msg = spambayes.message.SBHeaderMessage()
msg.setPayload(messageText)
msg.setId(state.getNewMessageName())
# Now find the spam disposition and add the header.
(prob, clues) = state.bayes.spamprob(msg.asTokens(),\
evidence=True)
msg.addSBHeaders(prob, clues)
# Check for "RETR" or "TOP N 99999999" - fetchmail without
# the 'fetchall' option uses the latter to retrieve messages.
if (command == 'RETR' or
(command == 'TOP' and
len(args) == 2 and args[1] == '99999999')):
cls = msg.GetClassification()
if cls == options["Headers", "header_ham_string"]:
state.numHams += 1
elif cls == options["Headers", "header_spam_string"]:
state.numSpams += 1
else:
state.numUnsure += 1
# Suppress caching of "Precedence: bulk" or
# "Precedence: list" ham if the options say so.
isSuppressedBulkHam = \
(cls == options["Headers", "header_ham_string"] and
options["pop3proxy", "no_cache_bulk_ham"] and
msg.get('precedence') in ['bulk', 'list'])
# Suppress large messages if the options say so.
size_limit = options["pop3proxy",
"no_cache_large_messages"]
isTooBig = size_limit > 0 and \
len(messageText) > size_limit
# Cache the message. Don't pollute the cache with test
# messages or suppressed bulk ham.
if (not state.isTest and
options["pop3proxy", "cache_messages"] and
not isSuppressedBulkHam and not isTooBig):
# Write the message into the Unknown cache.
message = state.unknownCorpus.makeMessage(msg.getId())
message.setSubstance(msg.as_string())
state.unknownCorpus.addMessage(message)
# We'll return the message with the headers added. We take
# all the headers from the SBHeaderMessage, but take the body
# directly from the POP3 conversation, because the
# SBHeaderMessage might have "fixed" a partial message by
# appending a closing boundary separator. Remember we can
# be dealing with partial message here because of the timeout
# code in onServerLine.
headers = []
for name, value in msg.items():
header = "%s: %s" % (name, value)
headers.append(re.sub(r'\r?\n', '\r\n', header))
body = re.split(r'\n\r?\n', messageText, 1)[1]
messageText = "\r\n".join(headers) + "\r\n\r\n" + body
except:
# Something nasty happened while parsing or classifying -
# report the exception in a hand-appended header and recover.
# This is one case where an unqualified 'except' is OK, 'cos
# anything's better than destroying people's email...
stream = cStringIO.StringIO()
traceback.print_exc(None, stream)
details = stream.getvalue()
# Build the header. This will strip leading whitespace from
# the lines, so we add a leading dot to maintain indentation.
detailLines = details.strip().split('\n')
dottedDetails = '\n.'.join(detailLines)
headerName = 'X-Spambayes-Exception'
header = Header(dottedDetails, header_name=headerName)
# Insert the header, converting email.Header's '\n' line
# breaks to POP3's '\r\n'.
headers, body = re.split(r'\n\r?\n', messageText, 1)
header = re.sub(r'\r?\n', '\r\n', str(header))
headers += "\n%s: %s\r\n\r\n" % (headerName, header)
messageText = headers + body
# Print the exception and a traceback.
print >>sys.stderr, details
# Restore the +OK and the POP3 .\r\n terminator if there was one.
retval = ok + "\n" + messageText
if terminatingDotPresent:
retval += '.\r\n'
return retval
else:
# Must be an error response.
return response
def onTop(self, command, args, response):
"""Adds the judgement header based on the raw headers and as
much of the body as the TOP command retrieves."""
# Easy (but see the caveat in BayesProxy.__doc__).
return self.onRetr(command, args, response)
def onUser(self, command, args, response):
"""Spins off three separate threads that expires any old messages
in the three caches, but does not do any processing of the USER
command itself."""
start_new_thread(state.spamCorpus.removeExpiredMessages, ())
start_new_thread(state.hamCorpus.removeExpiredMessages, ())
start_new_thread(state.unknownCorpus.removeExpiredMessages, ())
return response
def onUnknown(self, command, args, response):
"""Default handler; returns the server's response verbatim."""
return response
# This keeps the global state of the module - the command-line options,
# statistics like how many mails have been classified, the handle of the
# log file, the Classifier and FileCorpus objects, and so on.
class State:
def __init__(self):
"""Initialises the State object that holds the state of the app.
The default settings are read from Options.py and bayescustomize.ini
and are then overridden by the command-line processing code in the
__main__ code below."""
# Open the log file.
if options["globals", "verbose"]:
self.logFile = open('_pop3proxy.log', 'wb', 0)
self.servers = []
self.proxyPorts = []
if options["pop3proxy", "servers"]:
for server in options["pop3proxy", "servers"]:
server = server.strip()
if server.find(':') > -1:
server, port = server.split(':', 1)
else:
port = '110'
self.servers.append((server, int(port)))
if options["pop3proxy", "ports"]:
splitPorts = options["pop3proxy", "ports"]
self.proxyPorts = map(_addressAndPort, splitPorts)
if len(self.servers) != len(self.proxyPorts):
print "pop3proxy_servers & pop3proxy_ports are different lengths!"
sys.exit()
# Load up the other settings from Option.py / bayescustomize.ini
self.useDB = options["pop3proxy", "persistent_use_database"]
self.uiPort = options["html_ui", "port"]
self.launchUI = options["html_ui", "launch_browser"]
self.gzipCache = options["pop3proxy", "cache_use_gzip"]
self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"]
self.runTestServer = False
self.isTest = False
# Set up the statistics.
self.totalSessions = 0
self.activeSessions = 0
self.numSpams = 0
self.numHams = 0
self.numUnsure = 0
# Unique names for cached messages - see `getNewMessageName()` below.
self.lastBaseMessageName = ''
self.uniquifier = 2
def buildServerStrings(self):
"""After the server details have been set up, this creates string
versions of the details, for display in the Status panel."""
serverStrings = ["%s:%s" % (s, p) for s, p in self.servers]
self.serversString = ', '.join(serverStrings)
self.proxyPortsString = ', '.join(map(_addressPortStr, self.proxyPorts))
def createWorkers(self):
"""Using the options that were initialised in __init__ and then
possibly overridden by the driver code, create the Bayes object,
the Corpuses, the Trainers and so on."""
print "Loading database...",
if self.isTest:
self.useDB = True
options["pop3proxy", "persistent_storage_file"] = \
'_pop3proxy_test.pickle' # This is never saved.
filename = options["Storage", "persistent_storage_file"]
filename = os.path.expanduser(filename)
self.bayes = storage.open_storage(filename, self.useDB)
# Don't set up the caches and training objects when running the self-test,
# so as not to clutter the filesystem.
if not self.isTest:
def ensureDir(dirname):
try:
os.mkdir(dirname)
except OSError, e:
if e.errno != errno.EEXIST:
raise
# Create/open the Corpuses. Use small cache sizes to avoid hogging
# lots of memory.
map(ensureDir, [options["pop3proxy", "spam_cache"],
options["pop3proxy", "ham_cache"],
options["pop3proxy", "unknown_cache"]])
if self.gzipCache:
factory = GzipFileMessageFactory()
else:
factory = FileMessageFactory()
age = options["pop3proxy", "cache_expiry_days"]*24*60*60
self.spamCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"spam_cache"],
'[0123456789\-]*',
cacheSize=20)
self.hamCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"ham_cache"],
'[0123456789\-]*',
cacheSize=20)
self.unknownCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"unknown_cache"],
'[0123456789\-]*',
cacheSize=20)
# Given that (hopefully) users will get to the stage
# where they do not need to do any more regular training to
# be satisfied with spambayes' performance, we expire old
# messages from not only the trained corpora, but the unknown
# as well.
self.spamCorpus.removeExpiredMessages()
self.hamCorpus.removeExpiredMessages()
self.unknownCorpus.removeExpiredMessages()
# Create the Trainers.
self.spamTrainer = storage.SpamTrainer(self.bayes)
self.hamTrainer = storage.HamTrainer(self.bayes)
self.spamCorpus.addObserver(self.spamTrainer)
self.hamCorpus.addObserver(self.hamTrainer)
def getNewMessageName(self):
# The message name is the time it arrived, with a uniquifier
# appended if two arrive within one clock tick of each other.
messageName = "%10.10d" % long(time.time())
if messageName == self.lastBaseMessageName:
messageName = "%s-%d" % (messageName, self.uniquifier)
self.uniquifier += 1
else:
self.lastBaseMessageName = messageName
self.uniquifier = 2
return messageName
# Option-parsing helper functions
def _addressAndPort(s):
"""Decode a string representing a port to bind to, with optional address."""
s = s.strip()
if ':' in s:
addr, port = s.split(':')
return addr, int(port)
else:
return '', int(s)
def _addressPortStr((addr, port)):
"""Encode a string representing a port to bind to, with optional address."""
if not addr:
return str(port)
else:
return '%s:%d' % (addr, port)
state = State()
proxyListeners = []
def _createProxies(servers, proxyPorts):
"""Create BayesProxyListeners for all the given servers."""
for (server, serverPort), proxyPort in zip(servers, proxyPorts):
listener = BayesProxyListener(server, serverPort, proxyPort)
proxyListeners.append(listener)
def _recreateState():
global state
state = State()
# Close the existing listeners and create new ones. This won't
# affect any running proxies - once a listener has created a proxy,
# that proxy is then independent of it.
for proxy in proxyListeners:
proxy.close()
del proxyListeners[:]
prepare(state)
_createProxies(state.servers, state.proxyPorts)
return state
def main(servers, proxyPorts, uiPort, launchUI):
"""Runs the proxy forever or until a 'KILL' command is received or
someone hits Ctrl+Break."""
_createProxies(servers, proxyPorts)
httpServer = UserInterfaceServer(uiPort)
proxyUI = ProxyUserInterface(state, _recreateState)
httpServer.register(proxyUI)
Dibbler.run(launchBrowser=launchUI)
def prepare(state):
# Do whatever we've been asked to do...
state.createWorkers()
# Launch any SMTP proxies. Note that if the user hasn't specified any
# SMTP proxy information in their configuration, then nothing will
# happen.
import smtpproxy
servers, proxyPorts = smtpproxy.LoadServerInfo()
proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts,
state))
# setup info for the web interface
state.buildServerStrings()
def start(state):
# kick everything off
main(state.servers, state.proxyPorts, state.uiPort, state.launchUI)
def stop(state):
# Shutdown as though through the web UI. This will save the DB, allow
# any open proxy connections to complete, etc.
from urllib2 import urlopen
from urllib import urlencode
urlopen('http://localhost:%d/save' % state.uiPort,
urlencode({'how': 'Save & shutdown'})).read()
# ===================================================================
# __main__ driver.
# ===================================================================
def run():
# Read the arguments.
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbpsd:D:l:u:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
runSelfTest = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-b':
state.launchUI = True
elif opt == '-d': # dbm file
state.useDB = True
options["pop3proxy", "persistent_storage_file"] = arg
elif opt == '-D': # pickle file
state.useDB = False
options["pop3proxy", "persistent_storage_file"] = arg
elif opt == '-p': # dead option
print >>sys.stderr, "-p option is no longer supported, use -D\n"
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-l':
state.proxyPorts = [_addressAndPort(arg)]
elif opt == '-u':
state.uiPort = int(arg)
# Let the user know what they are using...
print get_version_string("POP3 Proxy")
print "and engine %s.\n" % (get_version_string(),)
prepare(state=state)
if 0 <= len(args) <= 2:
# Normal usage, with optional server name and port number.
if len(args) == 1:
state.servers = [(args[0], 110)]
elif len(args) == 2:
state.servers = [(args[0], int(args[1]))]
# Default to listening on port 110 for command-line-specified servers.
if len(args) > 0 and state.proxyPorts == []:
state.proxyPorts = [('', 110)]
start(state=state)
else:
print >>sys.stderr, __doc__
if __name__ == '__main__':
run()
--- NEW FILE: sb-smtpproxy.py ---
#!/usr/bin/env python
"""A SMTP proxy to train a Spambayes database.
You point SMTP Proxy at your SMTP server(s) and configure your email
client(s) to send mail through the proxy (i.e. usually this means you use
localhost as the outgoing server).
To setup, enter appropriate values in your Spambayes configuration file in
the "SMTP Proxy" section (in particular: "remote_servers", "listen_ports",
and "use_cached_message"). This configuration can also be carried out via
the web user interface offered by POP3 Proxy and IMAP Filter.
To use, simply forward/bounce mail that you wish to train to the
appropriate address (defaults to spambayes_spam@localhost and
spambayes_ham@localhost). All other mail is sent normally.
(Note that IMAP Filter and POP3 Proxy users should not execute this script;
launching of SMTP Proxy will be taken care of by those applicatons).
There are two main forms of operation. With both, mail to two
(user-configurable) email addresses is intercepted by the proxy (and is
*not* sent to the SMTP server) and used as training data for a Spambayes
database. All other mail is simply relayed to the SMTP server.
If the "use_cached_message" option is False, the proxy uses the message
sent as training data. This option is suitable for those not using
POP3 Proxy or IMAP Filter, or for those that are confident that their
mailer will forward/bounce messages in an unaltered form.
If the "use_cached_message" option is True, the proxy examines the message
for a unique spambayes identification number. It then tries to find this
message in the pop3proxy caches and on the imap servers. It then retrieves
the message from the cache/server and uses *this* as the training data.
This method is suitable for those using POP3 Proxy and/or IMAP Filter, and
avoids any potential problems with the mailer altering messages before
forwarding/bouncing them.
Usage:
smtpproxy [options]
note: option values with spaces must be enclosed in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-h : help
-v : verbose mode
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "Tim Stone, all the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
todo = """
o It would be nice if spam/ham could be bulk forwarded to the proxy,
rather than one by one. This would require separating the different
messages and extracting the correct ids. Simply changing to find
*all* the ids in a message, rather than stopping after one *might*
work, but I don't really know. Richie Hindle suggested something along
these lines back in September '02.
o Suggestions?
Testing:
o Test with as many clients as possible to check that the
id is correctly extracted from the forwarded/bounced message.
MUA information:
A '*' in the Header column signifies that the smtpproxy can extract
the id from the headers only. A '*' in the Body column signifies that
the smtpproxy can extract the id from the body of the message, if it
is there.
Header Body
*** Windows 2000 MUAs ***
Eudora 5.2 Forward * *
Eudora 5.2 Redirect *
Netscape Messenger (4.7) Forward (inline) * *
Netscape Messenger (4.7) Forward (quoted) Plain *
Netscape Messenger (4.7) Forward (quoted) HTML *
Netscape Messenger (4.7) Forward (quoted) Plain & HTML *
Netscape Messenger (4.7) Forward (attachment) Plain * *
Netscape Messenger (4.7) Forward (attachment) HTML * *
Netscape Messenger (4.7) Forward (attachment) Plain & HTML * *
Outlook Express 6 Forward HTML (Base64) *
Outlook Express 6 Forward HTML (None) *
Outlook Express 6 Forward HTML (QP) *
Outlook Express 6 Forward Plain (Base64) *
Outlook Express 6 Forward Plain (None) *
Outlook Express 6 Forward Plain (QP) *
Outlook Express 6 Forward Plain (uuencoded) *
http://www.endymion.com/products/mailman Forward *
M2 (Opera Mailer 7.01) Forward *
M2 (Opera Mailer 7.01) Redirect * *
The Bat! 1.62i Forward (RFC Headers not visible) *
The Bat! 1.62i Forward (RFC Headers visible) * *
The Bat! 1.62i Redirect *
The Bat! 1.62i Alternative Forward * *
The Bat! 1.62i Custom Template * *
AllegroMail 2.5.0.2 Forward *
AllegroMail 2.5.0.2 Redirect *
PocoMail 2.6.3 Bounce *
PocoMail 2.6.3 Bounce *
Pegasus Mail 4.02 Forward (all headers option set) * *
Pegasus Mail 4.02 Forward (all headers option not set) *
Calypso 3 Forward *
Calypso 3 Redirect * *
Becky! 2.05.10 Forward *
Becky! 2.05.10 Redirect *
Becky! 2.05.10 Redirect as attachment * *
Mozilla Mail 1.2.1 Forward (attachment) * *
Mozilla Mail 1.2.1 Forward (inline, plain) *1 *
Mozilla Mail 1.2.1 Forward (inline, plain & html) *1 *
Mozilla Mail 1.2.1 Forward (inline, html) *1 *
*1 The header method will only work if auto-include original message
is set, and if view all headers is true.
"""
import string
import re
import socket
import asyncore
import asynchat
import getopt
import sys
import os
from spambayes import Dibbler
from spambayes import storage
from spambayes.message import sbheadermessage_from_string
from spambayes.tokenizer import textparts
from spambayes.tokenizer import try_to_repair_damaged_base64
from spambayes.Options import options
from pop3proxy import _addressPortStr, ServerLineReader
from pop3proxy import _addressAndPort
class SMTPProxyBase(Dibbler.BrighterAsyncChat):
"""An async dispatcher that understands SMTP and proxies to a SMTP
server, calling `self.onTransaction(command, args)` for each
transaction.
self.onTransaction() should return the command to pass to
the proxied server - the command can be the verbatim command or a
processed version of it. The special command 'KILL' kills it (passing
a 'QUIT' command to the server).
"""
def __init__(self, clientSocket, serverName, serverPort):
Dibbler.BrighterAsyncChat.__init__(self, clientSocket)
self.request = ''
self.set_terminator('\r\n')
self.command = '' # The SMTP command being processed...
self.args = '' # ...and its arguments
self.isClosing = False # Has the server closed the socket?
self.inData = False
self.data = ""
self.blockData = False
self.serverSocket = ServerLineReader(serverName, serverPort,
self.onServerLine)
def onTransaction(self, command, args):
"""Overide this. Takes the raw command and returns the (possibly
processed) command to pass to the email client."""
raise NotImplementedError
def onProcessData(self, data):
"""Overide this. Takes the raw data and returns the (possibly
processed) data to pass back to the email client."""
raise NotImplementedError
def onServerLine(self, line):
"""A line of response has been received from the SMTP server."""
# Has the server closed its end of the socket?
if not line:
self.isClosing = True
# We don't process the return, just echo the response.
self.push(line)
self.onResponse()
def collect_incoming_data(self, data):
"""Asynchat override."""
self.request = self.request + data
def found_terminator(self):
"""Asynchat override."""
verb = self.request.strip().upper()
if verb == 'KILL':
self.socket.shutdown(2)
self.close()
raise SystemExit
if self.request.strip() == '':
# Someone just hit the Enter key.
self.command = self.args = ''
else:
# A proper command.
if self.request[:10].upper() == "MAIL FROM:":
splitCommand = self.request.split(":", 1)
elif self.request[:8].upper() == "RCPT TO:":
splitCommand = self.request.split(":", 1)
else:
splitCommand = self.request.strip().split(None, 1)
self.command = splitCommand[0]
self.args = splitCommand[1:]
if self.inData == True:
self.data += self.request + '\r\n'
if self.request == ".":
self.inData = False
cooked = self.onProcessData(self.data)
self.data = ""
if self.blockData == False:
self.serverSocket.push(cooked)
else:
self.push("250 OK\r\n")
else:
cooked = self.onTransaction(self.command, self.args)
if cooked is not None:
self.serverSocket.push(cooked + '\r\n')
self.command = self.args = self.request = ''
def onResponse(self):
# If onServerLine() decided that the server has closed its
# socket, close this one when the response has been sent.
if self.isClosing:
self.close_when_done()
# Reset.
self.command = ''
self.args = ''
self.isClosing = False
class BayesSMTPProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
BayesSMTPProxy objects to serve them."""
def __init__(self, serverName, serverPort, proxyPort, trainer):
proxyArgs = (serverName, serverPort, trainer)
Dibbler.Listener.__init__(self, proxyPort, BayesSMTPProxy,
proxyArgs)
print 'SMTP Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class BayesSMTPProxy(SMTPProxyBase):
"""Proxies between an email client and a SMTP server, inserting
judgement headers. It acts on the following SMTP commands:
o RCPT TO:
o Checks if the recipient address matches the key ham or spam
addresses, and if so notes this and does not forward a command to
the proxied server. In all other cases simply passes on the
verbatim command.
o DATA:
o Notes that we are in the data section. If (from the RCPT TO
information) we are receiving a ham/spam message to train on,
then do not forward the command on. Otherwise forward verbatim.
Any other commands are merely passed on verbatim to the server.
"""
def __init__(self, clientSocket, serverName, serverPort, trainer):
SMTPProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'RCPT TO': self.onRcptTo, 'DATA': self.onData,
'MAIL FROM': self.onMailFrom}
self.trainer = trainer
self.isClosed = False
self.train_as_ham = False
self.train_as_spam = False
def send(self, data):
try:
return SMTPProxyBase.send(self, data)
except socket.error:
# The email client has closed the connection - 40tude Dialog
# does this immediately after issuing a QUIT command,
# without waiting for the response.
self.close()
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
SMTPProxyBase.close(self)
def stripAddress(self, address):
"""
Strip the leading & trailing <> from an address. Handy for
getting FROM: addresses.
"""
if '<' in address:
start = string.index(address, '<') + 1
end = string.index(address, '>')
return address[start:end]
else:
return address
def splitTo(self, address):
"""Return 'address' as undressed (host, fulladdress) tuple.
Handy for use with TO: addresses."""
start = string.index(address, '<') + 1
sep = string.index(address, '@') + 1
end = string.index(address, '>')
return (address[sep:end], address[start:end],)
def onTransaction(self, command, args):
handler = self.handlers.get(command.upper(), self.onUnknown)
return handler(command, args)
def onProcessData(self, data):
if self.train_as_spam:
self.trainer.train(data, True)
self.train_as_spam = False
return ""
elif self.train_as_ham:
self.trainer.train(data, False)
self.train_as_ham = False
return ""
return data
def onRcptTo(self, command, args):
toHost, toFull = self.splitTo(args[0])
if toFull == options["smtpproxy", "spam_address"]:
self.train_as_spam = True
self.train_as_ham = False
self.blockData = True
self.push("250 OK\r\n")
return None
elif toFull == options["smtpproxy", "ham_address"]:
self.train_as_ham = True
self.train_as_spam = False
self.blockData = True
self.push("250 OK\r\n")
return None
else:
self.blockData = False
return "%s:%s" % (command, ' '.join(args))
def onData(self, command, args):
self.inData = True
if self.train_as_ham == True or self.train_as_spam == True:
self.push("250 OK\r\n")
return None
rv = command
for arg in args:
rv += ' ' + arg
return rv
def onMailFrom(self, command, args):
"""Just like the default handler, but has the necessary colon."""
rv = "%s:%s" % (command, ' '.join(args))
return rv
def onUnknown(self, command, args):
"""Default handler."""
return self.request
class SMTPTrainer(object):
def __init__(self, classifier, state=None, imap=None):
self.classifier = classifier
self.state = state
self.imap = imap
def extractSpambayesID(self, data):
msg = message_from_string(data)
# The nicest MUA is one that forwards the header intact.
id = msg.get(options["Headers", "mailid_header_name"])
if id is not None:
return id
# Some MUAs will put it in the body somewhere, while others will
# put it in an attached MIME message.
id = self._find_id_in_text(msg.as_string())
if id is not None:
return id
# the message might be encoded
for part in textparts(msg):
# Decode, or take it as-is if decoding fails.
try:
text = part.get_payload(decode=True)
except:
text = part.get_payload(decode=False)
if text is not None:
text = try_to_repair_damaged_base64(text)
if text is not None:
id = self._find_id_in_text(text)
return id
return None
header_pattern = re.escape(options["Headers", "mailid_header_name"])
# A MUA might enclose the id in a table, thus the convoluted re pattern
# (Mozilla Mail does this with inline html)
header_pattern += r":\s*(\\s*\\s*)?([\d\-]+)"
header_re = re.compile(header_pattern)
def _find_id_in_text(self, text):
mo = self.header_re.search(text)
if mo is None:
return None
return mo.group(2)
def train(self, msg, isSpam):
try:
use_cached = options["smtpproxy", "use_cached_message"]
except KeyError:
use_cached = True
if use_cached:
id = self.extractSpambayesID(msg)
if id is None:
print "Could not extract id"
return
self.train_cached_message(id, isSpam)
# Otherwise, train on the forwarded/bounced message.
msg = sbheadermessage_from_string(msg)
id = msg.setIdFromPayload()
msg.delSBHeaders()
if id is None:
# No id, so we don't have any reliable method of remembering
# information about this message, so we just assume that it
# hasn't been trained before. We could generate some sort of
# checksum for the message and use that as an id (this would
# mean that we didn't need to store the id with the message)
# but that might be a little unreliable.
self.classifier.learn(msg.asTokens(), isSpam)
else:
if msg.GetTrained() == (not isSpam):
self.classifier.unlearn(msg.asTokens(), not isSpam)
msg.RememberTrained(None)
if msg.GetTrained() is None:
self.classifier.learn(msg.asTokens(), isSpam)
msg.RememberTrained(isSpam)
def train_cached_message(self, id, isSpam):
if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
not self.train_message_on_imap_server(id, isSpam):
print "Could not find message (%s); perhaps it was " + \
"deleted from the POP3Proxy cache or the IMAP " + \
"server. This means that no training was done." % (id, )
def train_message_in_pop3proxy_cache(self, id, isSpam):
if self.state is None:
return False
sourceCorpus = None
for corpus in [self.state.unknownCorpus, self.state.hamCorpus,
self.state.spamCorpus]:
if corpus.get(id) is not None:
sourceCorpus = corpus
break
if corpus is None:
return False
if isSpam == True:
targetCorpus = self.state.spamCorpus
else:
targetCorpus = self.state.hamCorpus
targetCorpus.takeMessage(id, sourceCorpus)
self.classifier.store()
def train_message_on_imap_server(self, id, isSpam):
if self.imap is None:
return False
msg = self.imap.FindMessage(id)
if msg is None:
return False
if msg.GetTrained() == (not isSpam):
msg.get_substance()
msg.delSBHeaders()
self.classifier.unlearn(msg.asTokens(), not isSpam)
msg.RememberTrained(None)
if msg.GetTrained() is None:
msg.get_substance()
msg.delSBHeaders()
self.classifier.learn(msg.asTokens(), isSpam)
msg.RememberTrained(isSpam)
def LoadServerInfo():
# Load the proxy settings
servers = []
proxyPorts = []
if options["smtpproxy", "remote_servers"]:
for server in options["smtpproxy", "remote_servers"]:
server = server.strip()
if server.find(':') > -1:
server, port = server.split(':', 1)
else:
port = '25'
servers.append((server, int(port)))
if options["smtpproxy", "listen_ports"]:
splitPorts = options["smtpproxy", "listen_ports"]
proxyPorts = map(_addressAndPort, splitPorts)
if len(servers) != len(proxyPorts):
print "smtpproxy:remote_servers & smtpproxy:listen_ports are " + \
"different lengths!"
sys.exit()
return servers, proxyPorts
def CreateProxies(servers, proxyPorts, trainer):
"""Create BayesSMTPProxyListeners for all the given servers."""
# allow for old versions of pop3proxy
if not isinstance(trainer, SMTPTrainer):
trainer = SMTPTrainer(trainer.bayes, trainer)
proxyListeners = []
for (server, serverPort), proxyPort in zip(servers, proxyPorts):
listener = BayesSMTPProxyListener(server, serverPort, proxyPort,
trainer)
proxyListeners.append(listener)
return proxyListeners
def main():
"""Runs the proxy until a 'KILL' command is received or someone hits
Ctrl+Break."""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hvd:D:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = options["Storage", "persistent_storage_file"]
useDBM = options["Storage", "persistent_use_database"]
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == '-v':
options["globals", "verbose"] = True
bdbname = os.path.expanduser(bdbname)
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
if useDBM:
classifier = storage.DBDictClassifier(bdbname)
else:
classifier = storage.PickledClassifier(bdbname)
if options["globals", "verbose"]:
print "Done."
servers, proxyPorts = LoadServerInfo()
trainer = SMTPTrainer(classifier)
proxyListeners = CreateProxies(servers, proxyPorts, trainer)
Dibbler.run()
if __name__ == '__main__':
main()
--- NEW FILE: sb-unheader.py ---
#!/usr/bin/env python
"""
unheader.py: cleans headers from email messages. By default, this
removes SpamAssassin headers, specify a pattern with -p to supply
new headers to remove.
This is often needed because existing spamassassin headers can
provide killer spam clues, for all the wrong reasons.
"""
import re
import sys
import os
import glob
import mailbox
import email.Parser
import email.Message
import email.Generator
import getopt
def unheader(msg, pat):
pat = re.compile(pat)
for hdr in msg.keys():
if pat.match(hdr):
del msg[hdr]
# remain compatible with 2.2.1 - steal replace_header from 2.3 source
class Message(email.Message.Message):
def replace_header(self, _name, _value):
"""Replace a header.
Replace the first matching header found in the message, retaining
header order and case. If no matching header was found, a
KeyError is raised.
"""
_name = _name.lower()
for i, (k, v) in zip(range(len(self._headers)), self._headers):
if k.lower() == _name:
self._headers[i] = (k, _value)
break
else:
raise KeyError, _name
class Parser(email.Parser.HeaderParser):
def __init__(self):
email.Parser.Parser.__init__(self, Message)
def deSA(msg):
if msg['X-Spam-Status']:
if msg['X-Spam-Status'].startswith('Yes'):
pct = msg['X-Spam-Prev-Content-Type']
if pct:
msg['Content-Type'] = pct
pcte = msg['X-Spam-Prev-Content-Transfer-Encoding']
if pcte:
msg['Content-Transfer-Encoding'] = pcte
subj = re.sub(r'\*\*\*\*\*SPAM\*\*\*\*\* ', '',
msg['Subject'] or "")
if subj != msg["Subject"]:
msg.replace_header("Subject", subj)
body = msg.get_payload()
newbody = []
at_start = 1
for line in body.splitlines():
if at_start and line.startswith('SPAM: '):
continue
elif at_start:
at_start = 0
newbody.append(line)
msg.set_payload("\n".join(newbody))
unheader(msg, "X-Spam-")
def process_message(msg, dosa, pats):
if pats is not None:
unheader(msg, pats)
if dosa:
deSA(msg)
def process_mailbox(f, dosa=1, pats=None):
gen = email.Generator.Generator(sys.stdout, maxheaderlen=0)
for msg in mailbox.PortableUnixMailbox(f, Parser().parse):
process_message(msg, dosa, pats)
gen(msg, unixfrom=1)
def process_maildir(d, dosa=1, pats=None):
parser = Parser()
for fn in glob.glob(os.path.join(d, "cur", "*")):
print ("reading from %s..." % fn),
file = open(fn)
msg = parser.parse(file)
process_message(msg, dosa, pats)
tmpfn = os.path.join(d, "tmp", os.path.basename(fn))
tmpfile = open(tmpfn, "w")
print "writing to %s" % tmpfn
email.Generator.Generator(tmpfile, maxheaderlen=0)(msg, unixfrom=0)
os.rename(tmpfn, fn)
def usage():
print >> sys.stderr, "usage: unheader.py [ -p pat ... ] [ -s ] folder"
print >> sys.stderr, "-p pat gives a regex pattern used to eliminate unwanted headers"
print >> sys.stderr, "'-p pat' may be given multiple times"
print >> sys.stderr, "-s tells not to remove SpamAssassin headers"
print >> sys.stderr, "-d means treat folder as a Maildir"
def main(args):
headerpats = []
dosa = 1
ismbox = 1
try:
opts, args = getopt.getopt(args, "p:shd")
except getopt.GetoptError:
usage()
sys.exit(1)
else:
for opt, arg in opts:
if opt == "-h":
usage()
sys.exit(0)
elif opt == "-p":
headerpats.append(arg)
elif opt == "-s":
dosa = 0
elif opt == "-d":
ismbox = 0
pats = headerpats and "|".join(headerpats) or None
if len(args) != 1:
usage()
sys.exit(1)
if ismbox:
f = file(args[0])
process_mailbox(f, dosa, pats)
else:
process_maildir(args[0], dosa, pats)
if __name__ == "__main__":
main(sys.argv[1:])
--- NEW FILE: sb-upload.py ---
#!/usr/bin/env python
"""
Read a message or a mailbox file on standard input, upload it to a
web browser and write it to standard output.
usage: %(progname)s [-h] [-n] [-s server] [-p port] [-r N]
Options:
-h, --help - print help and exit
-n, --null - suppress writing to standard output (default %(null)s)
-s, --server= - provide alternate web server (default %(server)s)
-p, --port= - provide alternate server port (default %(port)s)
-r, --prob= - feed the message to the trainer w/ prob N [0.0...1.0]
"""
import sys
import httplib
import mimetypes
import getopt
import random
from spambayes.Options import options
progname = sys.argv[0]
__author__ = "Skip Montanaro "
__credits__ = "Spambayes gang, Wade Leftwich"
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
# appropriated verbatim from a recipe by Wade Leftwich in the Python
# Cookbook: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
def post_multipart(host, selector, fields, files):
"""
Post fields and files to an http host as multipart/form-data. fields is
a sequence of (name, value) elements for regular form fields. files is
a sequence of (name, filename, value) elements for data to be uploaded
as files. Return the server's response page.
"""
content_type, body = encode_multipart_formdata(fields, files)
h = httplib.HTTP(host)
h.putrequest('POST', selector)
h.putheader('content-type', content_type)
h.putheader('content-length', str(len(body)))
h.endheaders()
h.send(body)
errcode, errmsg, headers = h.getreply()
return h.file.read()
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be
uploaded as files. Return (content_type, body) ready for httplib.HTTP
instance
"""
BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def usage(*args):
defaults = {}
for d in args:
defaults.update(d)
print __doc__ % defaults
def main(argv):
null = False
server = "localhost"
port = options["html_ui", "port"]
prob = 1.0
try:
opts, args = getopt.getopt(argv, "hns:p:r:",
["help", "null", "server=", "port=",
"prob="])
except getopt.error:
usage(globals(), locals())
sys.exit(1)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage(globals(), locals())
sys.exit(0)
elif opt in ("-n", "--null"):
null = True
elif opt in ("-s", "--server"):
server = arg
elif opt in ("-p", "--port"):
port = int(arg)
elif opt in ("-r", "--prob"):
n = float(arg)
if n < 0.0 or n > 1.0:
usage(globals(), locals())
sys.exit(1)
prob = n
if args:
usage(globals(), locals())
sys.exit(1)
data = sys.stdin.read()
sys.stdout.write(data)
if random.random() < prob:
try:
post_multipart("%s:%d"%(server,port), "/upload", [],
[('file', 'message.dat', data)])
except:
# not an error if the server isn't responding
pass
if __name__ == "__main__":
main(sys.argv[1:])
--- NEW FILE: sb-xmlrpcserver.py ---
#! /usr/bin/env python
# A server version of hammie.py
"""Usage: %(program)s [options] IP:PORT
Where:
-h
show usage and exit
-p FILE
use file as the persistent store. loads data from this file if it
exists, and saves data to this file at the end. Default: %(DEFAULTDB)s
-d
use the DBM store instead of cPickle. The file is larger and
creating it is slower, but checking against it is much faster,
especially for large word databases.
IP
IP address to bind (use 0.0.0.0 to listen on all IPs of this machine)
PORT
Port number to listen to.
"""
import SimpleXMLRPCServer
import getopt
import sys
import traceback
import xmlrpclib
from spambayes import hammie
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
program = sys.argv[0] # For usage(); referenced by docstring above
# Default DB path
DEFAULTDB = hammie.DEFAULTDB
class XMLHammie(hammie.Hammie):
def score(self, msg, *extra):
try:
msg = msg.data
except AttributeError:
pass
return xmlrpclib.Binary(hammie.Hammie.score(self, msg, *extra))
def filter(self, msg, *extra):
try:
msg = msg.data
except AttributeError:
pass
return xmlrpclib.Binary(hammie.Hammie.filter(self, msg, *extra))
class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
def do_POST(self):
"""Handles the HTTP POST request.
Attempts to interpret all HTTP POST requests as XML-RPC calls,
which are forwarded to the _dispatch method for handling.
This one also prints out tracebacks, to help me debug :)
"""
try:
# get arguments
data = self.rfile.read(int(self.headers["content-length"]))
params, method = xmlrpclib.loads(data)
# generate response
try:
response = self._dispatch(method, params)
# wrap response in a singleton tuple
response = (response,)
except:
traceback.print_exc()
# report exception back to server
response = xmlrpclib.dumps(
xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
)
else:
response = xmlrpclib.dumps(response, methodresponse=1)
except:
# internal error, report as HTTP server error
traceback.print_exc()
print `data`
self.send_response(500)
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def main():
"""Main program; parse options and go."""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hdp:')
except getopt.error, msg:
usage(2, msg)
pck = DEFAULTDB
usedb = False
for opt, arg in opts:
if opt == '-h':
usage(0)
elif opt == '-p':
pck = arg
elif opt == "-d":
usedb = True
if len(args) != 1:
usage(2, "IP:PORT not specified")
ip, port = args[0].split(":")
port = int(port)
bayes = hammie.createbayes(pck, usedb)
h = XMLHammie(bayes)
server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler)
server.register_instance(h)
server.serve_forever()
if __name__ == "__main__":
main()
--- sb-overkill.py DELETED ---
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:30 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:15:34 2003
Subject: [Spambayes-checkins] spambayes/contrib nway.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/contrib
In directory sc8-pr-cvs1:/tmp/cvs-serv14102/contrib
Modified Files:
nway.py
Log Message:
Remove backwards compat code for options, and update all (I hope)
the remaining code that uses it.
Index: nway.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/contrib/nway.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** nway.py 13 Aug 2003 14:39:35 -0000 1.2
--- nway.py 5 Sep 2003 01:15:28 -0000 1.3
***************
*** 62,66 ****
h = hammie.open(db, True, 'r')
score = h.score(msg)
! if score >= Options.options.spam_cutoff:
msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
break
--- 62,66 ----
h = hammie.open(db, True, 'r')
score = h.score(msg)
! if score >= Options.options["Categorization", "spam_cutoff"]:
msg["X-Spambayes-Classification"] = "%s; %.2f" % (tag, score)
break
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:15:38 2003
Subject: [Spambayes-checkins]
spambayes/spambayes/resources __init__.py, 1.1, 1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv14102/spambayes/resources
Modified Files:
__init__.py
Log Message:
Remove backwards compat code for options, and update all (I hope)
the remaining code that uses it.
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/__init__.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** __init__.py 17 Jan 2003 20:21:13 -0000 1.1
--- __init__.py 5 Sep 2003 01:15:28 -0000 1.2
***************
*** 1,2 ****
! # See scanning__init__.py for how to change these resources.
--- 1,36 ----
+ """Design-time __init__.py for resourcepackage
! This is the scanning version of __init__.py for your
! resource modules. You replace it with a blank or doc-only
! init when ready to release.
! """
! try:
! __file__
! except NameError:
! pass
! else:
! import os
! if os.path.splitext(os.path.basename( __file__ ))[0] == "__init__":
! try:
! from resourcepackage import package, defaultgenerators
! generators = defaultgenerators.generators.copy()
!
! ### CUSTOMISATION POINT
! ## import specialised generators here, such as for wxPython
! #from resourcepackage import wxgenerators
! #generators.update( wxgenerators.generators )
! except ImportError:
! pass
! else:
! package = package.Package(
! packageName = __name__,
! directory = os.path.dirname( os.path.abspath(__file__) ),
! generators = generators,
! )
! package.scan(
! ### CUSTOMISATION POINT
! ## force true -> always re-loads from external files, otherwise
! ## only reloads if the file is newer than the generated .py file.
! # force = 1,
! )
!
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:15:42 2003
Subject: [Spambayes-checkins] spambayes/testtools incremental.py, 1.4,
1.5 simplexloop.py, 1.3, 1.4 timcv.py, 1.3, 1.4 timtest.py,
1.4, 1.5 weaktest.py, 1.3, 1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/testtools
In directory sc8-pr-cvs1:/tmp/cvs-serv14102/testtools
Modified Files:
incremental.py simplexloop.py timcv.py timtest.py weaktest.py
Log Message:
Remove backwards compat code for options, and update all (I hope)
the remaining code that uses it.
Index: incremental.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/testtools/incremental.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** incremental.py 28 Feb 2003 17:57:53 -0000 1.4
--- incremental.py 5 Sep 2003 01:15:29 -0000 1.5
***************
*** 111,116 ****
for example in stream:
prob = guess(example)
! is_ham_guessed = prob < options.ham_cutoff
! is_spam_guessed = prob >= options.spam_cutoff
if is_spam:
self.nspam_tested += 1
--- 111,116 ----
for example in stream:
prob = guess(example)
! is_ham_guessed = prob < options["Categorization", "ham_cutoff"]
! is_spam_guessed = prob >= options["Categorization", "spam_cutoff"]
if is_spam:
self.nspam_tested += 1
***************
*** 176,180 ****
>>> from spambayes.classifier import Bayes
>>> from spambayes.Options import options
! >>> options.ham_cutoff = options.spam_cutoff = 0.5
>>> good1 = _Example('', ['a', 'b', 'c'])
--- 176,180 ----
>>> from spambayes.classifier import Bayes
>>> from spambayes.Options import options
! >>> options["Categorization", "ham_cutoff"] = options["Categorization", "spam_cutoff"] = 0.5
>>> good1 = _Example('', ['a', 'b', 'c'])
Index: simplexloop.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/testtools/simplexloop.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** simplexloop.py 29 Jan 2003 03:23:35 -0000 1.3
--- simplexloop.py 5 Sep 2003 01:15:29 -0000 1.4
***************
*** 33,39 ****
from spambayes import Options
! start = (Options.options.unknown_word_prob,
! Options.options.minimum_prob_strength,
! Options.options.unknown_word_strength)
err = (0.01, 0.01, 0.01)
--- 33,39 ----
from spambayes import Options
! start = (Options.options["Tokenizer", "unknown_word_prob",
! Options.options["Tokenzier", "minimum_prob_strength"],
! Options.options["Tokenizer", "unknown_word_strength"])
err = (0.01, 0.01, 0.01)
Index: timcv.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/testtools/timcv.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** timcv.py 8 Mar 2003 00:33:23 -0000 1.3
--- timcv.py 5 Sep 2003 01:15:29 -0000 1.4
***************
*** 73,78 ****
print options.display()
! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)]
! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)]
d = TestDriver.Driver()
--- 73,80 ----
print options.display()
! hamdirs = [options["TestDriver", "ham_directories"] % \
! i for i in range(1, nsets+1)]
! spamdirs = [options["TestDriver", "spam_directories"] % \
! i for i in range(1, nsets+1)]
d = TestDriver.Driver()
***************
*** 91,95 ****
if i > 0:
! if options.build_each_classifier_from_scratch:
# Build a new classifier from the other sets.
d.new_classifier()
--- 93,97 ----
if i > 0:
! if options["CV Driver", "build_each_classifier_from_scratch"]:
# Build a new classifier from the other sets.
d.new_classifier()
***************
*** 114,118 ****
d.finishtest()
! if i < nsets - 1 and not options.build_each_classifier_from_scratch:
# Add this set back in.
d.train(hamstream, spamstream)
--- 116,121 ----
d.finishtest()
! if i < nsets - 1 and not options["CV Driver",
! "build_each_classifier_from_scratch"]:
# Add this set back in.
d.train(hamstream, spamstream)
Index: timtest.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/testtools/timtest.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** timtest.py 30 Apr 2003 06:43:28 -0000 1.4
--- timtest.py 5 Sep 2003 01:15:29 -0000 1.5
***************
*** 58,63 ****
print options.display()
! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)]
! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)]
spamhamdirs = zip(spamdirs, hamdirs)
--- 58,65 ----
print options.display()
! spamdirs = [options["TestDriver", "spam_directories"] % \
! i for i in range(1, nsets+1)]
! hamdirs = [options["TestDriver", "ham_directories"] % \
! i for i in range(1, nsets+1)]
spamhamdirs = zip(spamdirs, hamdirs)
Index: weaktest.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/testtools/weaktest.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** weaktest.py 29 Jan 2003 03:23:35 -0000 1.3
--- weaktest.py 5 Sep 2003 01:15:29 -0000 1.4
***************
*** 62,79 ****
class UnsureAndFalses(TrainDecision):
def spamtrain(self,scr):
! if scr < options.spam_cutoff:
return TRAIN_AS_SPAM
def hamtrain(self,scr):
! if scr > options.ham_cutoff:
return TRAIN_AS_HAM
class UnsureOnly(TrainDecision):
def spamtrain(self,scr):
! if options.ham_cutoff < scr < options.spam_cutoff:
return TRAIN_AS_SPAM
def hamtrain(self,scr):
! if options.ham_cutoff < scr < options.spam_cutoff:
return TRAIN_AS_HAM
--- 62,81 ----
class UnsureAndFalses(TrainDecision):
def spamtrain(self,scr):
! if scr < options["Categorization", "spam_cutoff"]:
return TRAIN_AS_SPAM
def hamtrain(self,scr):
! if scr > options["Categorization", "ham_cutoff"]:
return TRAIN_AS_HAM
class UnsureOnly(TrainDecision):
def spamtrain(self,scr):
! if options["Categorization", "ham_cutoff"] < scr < \
! options["Categorization", "spam_cutoff"]:
return TRAIN_AS_SPAM
def hamtrain(self,scr):
! if options["Categorization", "ham_cutoff"] < scr < \
! options["Categorization", "spam_cutoff"]:
return TRAIN_AS_HAM
***************
*** 96,102 ****
class OwnDecision(TrainDecision):
def hamtrain(self,scr):
! if scr < options.ham_cutoff:
return TRAIN_AS_HAM
! elif scr > options.spam_cutoff:
return TRAIN_AS_SPAM
--- 98,104 ----
class OwnDecision(TrainDecision):
def hamtrain(self,scr):
! if scr < options["Categorization", "ham_cutoff"]:
return TRAIN_AS_HAM
! elif scr > options["Categorization", "spam_cutoff"]:
return TRAIN_AS_SPAM
***************
*** 146,151 ****
print options.display()
! spamdirs = [options.spam_directories % i for i in range(1, nsets+1)]
! hamdirs = [options.ham_directories % i for i in range(1, nsets+1)]
spamfns = [(x,y,1) for x in spamdirs for y in os.listdir(x)]
--- 148,155 ----
print options.display()
! spamdirs = [options["TestDriver", "spam_directories"] % \
! i for i in range(1, nsets+1)]
! hamdirs = [options["TestDriver", "ham_directories"] % \
! i for i in range(1, nsets+1)]
spamfns = [(x,y,1) for x in spamdirs for y in os.listdir(x)]
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:15:46 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv14102/windows
Modified Files:
autoconfigure.py
Log Message:
Remove backwards compat code for options, and update all (I hope)
the remaining code that uses it.
Index: autoconfigure.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** autoconfigure.py 2 Sep 2003 08:29:01 -0000 1.3
--- autoconfigure.py 5 Sep 2003 01:15:29 -0000 1.4
***************
*** 55,63 ****
--- 55,70 ----
import re
import os
+ import sys
import types
import socket
+ import shutil
import StringIO
import ConfigParser
+ # Allow for those without SpamBayes on the PYTHONPATH
+ sys.path.insert(-1, os.getcwd())
+ sys.path.insert(-1, os.path.dirname(os.getcwd()))
+
+ from spambayes import OptionsClass
from spambayes.Options import options, optionsPathname
***************
*** 75,79 ****
s.close()
except socket.error:
! return port
# Let's be safe and use high ports, starting at 1110 and 1025, and going up
--- 82,91 ----
s.close()
except socket.error:
! portStr = str(port)
! if options["pop3proxy", "listen_ports"].find(portStr) != -1 or \
! options["smtpproxy", "listen_ports"].find(portStr) != -1:
! continue
! else:
! return port
# Let's be safe and use high ports, starting at 1110 and 1025, and going up
***************
*** 470,475 ****
--- 482,660 ----
# option, obviously)
+ def configure_pegasus_mail(config_location):
+ """Configure Pegasus Mail to use the SpamBayes POP3 and SMTP proxies,
+ and configure SpamBayes to proxy the servers that Pegasus Mail was
+ connecting to."""
+
+ # We can't use ConfigParser here, as we want 'surgical' editing,
+ # so we want to use out OptionsClass. There is the additional trouble
+ # that the Pegasus Mail config file doesn't have a section header.
+
+ pop_proxy = pop_proxy_port
+ smtp_proxy = smtp_proxy_port
+
+ for filename in os.listdir(config_location):
+ if filename.lower().startswith("pop") or filename.lower().startswith("smt"):
+ full_filename = os.path.join(config_location, filename)
+ working_filename = "%s.tmp" % (filename, )
+ shutil.copyfile(filename, working_filename)
+ c = OptionsClass.OptionsClass()
+ c.merge_file(working_filename)
+ server = "%s:%s" % (c.get("all", "host"), c.get("all", "port"))
+ if filename[:3] == "pop":
+ pop_proxy = move_to_next_free_port(pop_proxy)
+ proxy = pop_proxy
+ sect = "pop3proxy"
+ else:
+ smtp_proxy = move_to_next_free_port(smtp_proxy)
+ proxy = smtp_proxy
+ sect = "smtpproxy"
+ options[sect, "remote_servers"] += (server,)
+ options[sect, "listen_ports"] += (proxy,)
+ # Write in the new options!!
+ c.set("all", "host", "127.0.0.1")
+ c.set("all", "port", proxy)
+ c.update_file(working_filename)
+ if options["globals", "verbose"]:
+ print "[%s] Proxy %s on localhost:%s" % \
+ (c.get("all", "title"), server, proxy)
+ elif filename.lower() == "IMAP.PM":
+ # Setup imapfilter instead.
+ pass
+
+ # Pegasus Mail has a 'weight' system for determining junk mail.
+ # The best plan would probably be to just add to this. Something like:
+ rules_filename = os.path.join(config_location, "spambust.dat")
+ header_name = options["Headers", "classification_header_name"]
+ spam_tag = options["Headers", "header_spam_string"]
+ unsure_tag = options["Headers", "header_unsure_string"]
+ ham_tag = options["Headers", "header_ham_string"]
+ spam_weight = 500
+ ham_weight = -500
+ unsure_weight = -50 # leave judgement up to the rest of the rules
+ rule = '# SpamBayes adjustments\n' \
+ 'if header "%s" contains "%s" weight %s\n' \
+ 'if header "%s" contains "%s" weight %s\n' \
+ 'if header "%s" contains "%s" wieght %s\n\n' % \
+ (header_name, spam_tag, spam_weight,
+ header_name, unsure_tag, unsure_weight,
+ header_name, ham_tag, ham_weight)
+ rules_file = file(rules_filename, "a")
+ rules_file.write(rule)
+ rules_file.close()
+
+ def configure_pocomail():
+ import win32api
+ import win32con
+
+ key = "Software\\Poco Systems Inc"
+
+ pop_proxy = pop_proxy_port
+ smtp_proxy = smtp_proxy_port
+
+ reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, key)
+ subkey_name = "%s\\%s" % (key, win32api.RegEnumKey(reg, 0))
+ reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER,
+ subkey_name)
+ pocomail_path = win32api.RegQueryValueEx(reg, "Path")[0]
+
+ pocomail_accounts_file = os.path.join(pocomail_path, "accounts.ini")
+
+ if os.path.exists(pocomail_accounts_file):
+ f = open(pocomail_accounts_file, "r")
+
+ accountName = ""
+ pocomail_accounts = { }
+
+ # Builds the dictionary with all the existing accounts.
+ for line in f.readlines():
+ line = line.rstrip('\n\r')
+ if line == '':
+ continue
+
+ if line[0] == '[' and line[-1] == ']':
+ accountName = line[1:-1]
+ pocomail_accounts[accountName] = { }
+ else:
+ separator = line.find('=')
+ optionName = line[:separator]
+ optionValue = line[separator + 1:]
+
+ if optionName == "POPServer":
+ pop3 = optionValue.split(':')
+ if len(pop3) == 1:
+ pop3.append(110)
+ server = "%s:%s" % tuple(pop3)
+
+ proxy = pop_proxy
+ pop_proxy = move_to_next_free_port(pop_proxy)
+
+ if not server in options["pop3proxy", "remote_servers"]:
+ options["pop3proxy", "remote_servers"] += (server,)
+ options["pop3proxy", "listen_ports"] += (proxy,)
+ else:
+ serverIndex = 0
+ for remoteServer in options["pop3proxy",
+ "remote_servers"]:
+ if remoteServer == server:
+ break
+ serverIndex += 1
+ proxy = options["pop3proxy", "listen_ports"][serverIndex]
+
+ optionValue = "%s:%s" % ('localhost', proxy)
+
+ pocomail_accounts[accountName][optionName] = optionValue
+
+ f.close()
+ f = open(pocomail_accounts_file, "w")
+ for accountName in pocomail_accounts.keys():
+ f.write('[' + accountName + ']\n')
+ for optionName, optionValue in pocomail_accounts[accountName].items():
+ f.write("%s=%s\n" % (optionName, optionValue))
+ f.write('\n')
+ f.close()
+
+ options.update_file(optionsPathname)
+
+ # Add a filter to pocomail
+ pocomail_filters_file = os.path.join(pocomail_path, "filters.ini")
+
+ if os.path.exists(pocomail_filters_file):
+ f = open(pocomail_filters_file, "r")
+
+ pocomail_filters = { }
+ filterName = ""
+
+ for line in f.readlines():
+ line = line.rstrip('\n\r')
+ if line == '': continue
+
+ if line[0] == '[' and line[-1] == ']':
+ filterName = line[1:-1]
+ pocomail_filters[filterName] = []
+ elif line[0] != '{':
+ pocomail_filters[filterName].append(line)
+ f.close()
+
+ spamBayesFilter = 'spam,X-Spambayes-Classification,move,' \
+ '"Junk Mail",0,0,,,0,,,move,In,0,0,,0,,,' \
+ 'move,In,0,0,,0,,,move,In,0,0,,0,,,move,' \
+ 'In,0,0,,0,,,move,In,0,0,1,0'
+ if pocomail_filters.has_key("Incoming") and \
+ spamBayesFilter not in pocomail_filters["Incoming"]:
+ pocomail_filters["Incoming"].append(spamBayesFilter)
+
+ f = open(pocomail_filters_file, "w")
+ f.write('{ Filter list generated by PocoMail 3.01 (1661)' \
+ '- Licensed Version}\n')
+ for filterName in pocomail_filters.keys():
+ f.write('\n[' + filterName + ']\n')
+ for filter in pocomail_filters[filterName]:
+ f.write(filter + '\n')
+ f.close()
+
if __name__ == "__main__":
+ pmail_ini_dir = "C:\\Program Files\\PMAIL\\MAIL\\ADMIN"
# XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224"
# XXX but I presume it's different for everyone? I'll have to check on
***************
*** 478,481 ****
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
! configure_outlook_express()
pass
--- 663,668 ----
#configure_mozilla(mozilla_ini_dir)
#configure_m2(m2_ini_dir)
! #configure_outlook_express()
! #configure_pocomail()
! configure_pegasus_mail(pmail_ini_dir)
pass
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:15:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:15:50 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.69,
1.70 OptionsClass.py, 1.11, 1.12 TestDriver.py, 1.3,
1.4 classifier.py, 1.8, 1.9 dbmstorage.py, 1.7, 1.8 hammie.py,
1.7, 1.8 storage.py, 1.27, 1.28 tokenizer.py, 1.13, 1.14
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv14102/spambayes
Modified Files:
Options.py OptionsClass.py TestDriver.py classifier.py
dbmstorage.py hammie.py storage.py tokenizer.py
Log Message:
Remove backwards compat code for options, and update all (I hope)
the remaining code that uses it.
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.69
retrieving revision 1.70
diff -C2 -d -r1.69 -r1.70
*** Options.py 1 Sep 2003 06:07:27 -0000 1.69
--- Options.py 5 Sep 2003 01:15:28 -0000 1.70
***************
*** 15,21 ****
To Do:
! o Get rid of the really ugly backwards compatability code (that adds
! many, many attributes to the options object) as soon as all the
! modules are changed over.
"""
--- 15,19 ----
To Do:
! o Suggestions?
"""
***************
*** 24,78 ****
__all__ = ['options']
- # Backwards compatibility stuff - this will be removed at some point
- # This table allows easy conversion from the (old_section, old_option)
- # names to the (new_section, new_option) names.
- conversion_table = {("Hammie", "clue_mailheader_cutoff"):
- ("Headers", "clue_mailheader_cutoff"),
- ("Hammie", "header_score_digits"):
- ("Headers", "header_score_digits"),
- ("Hammie", "header_score_logarithm"):
- ("Headers", "header_score_logarithm"),
- ("Hammie", "header_name"):
- ("Headers", "classification_header_name"),
- ("Hammie", "header_ham_string"):
- ("Headers", "header_ham_string"),
- ("Hammie", "header_spam_string"):
- ("Headers", "header_spam_string"),
- ("Hammie", "header_unsure_string"):
- ("Headers", "header_unsure_string"),
- ("Hammie", "trained_header"):
- ("Headers", "trained_header_name"),
- ("pop3proxy", "evidence_header_name"):
- ("Headers", "evidence_header_name"),
- ("pop3proxy", "mailid_header_name"):
- ("Headers", "mailid_header_name"),
- ("pop3proxy", "prob_header_name"):
- ("Headers", "score_header_name"),
- ("pop3proxy", "thermostat_header_name"):
- ("Headers", "thermostat_header_name"),
- ("pop3proxy", "include_evidence"):
- ("Headers", "include_evidence"),
- ("pop3proxy", "include_prob"):
- ("Headers", "include_score"),
- ("pop3proxy", "include_thermostat"):
- ("Headers", "include_thermostat"),
- ("pop3proxy", "ports"):
- ("pop3proxy", "listen_ports"),
- ("pop3proxy", "servers"):
- ("pop3proxy", "remote_servers"),
- ("smtpproxy", "ports"):
- ("smtpproxy", "listen_ports"),
- ("smtpproxy", "servers"):
- ("smtpproxy", "remote_servers"),
- ("hammiefilter", "persistent_storage_file"):
- ("Storage", "persistent_storage_file"),
- ("hammiefilter", "persistent_use_database"):
- ("Storage", "persistent_use_database"),
- ("pop3proxy", "persistent_storage_file"):
- ("Storage", "persistent_storage_file"),
- ("pop3proxy", "persistent_use_database"):
- ("Storage", "persistent_use_database"),
- }
-
# Grab the stuff from the core options class.
from OptionsClass import *
--- 22,25 ----
***************
*** 983,987 ****
options = OptionsClass()
- options.conversion_table = conversion_table # Our b/w compat hacks
options.load_defaults(defaults)
--- 930,933 ----
Index: OptionsClass.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/OptionsClass.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** OptionsClass.py 25 Aug 2003 09:00:36 -0000 1.11
--- OptionsClass.py 5 Sep 2003 01:15:28 -0000 1.12
***************
*** 23,31 ****
To Do:
! o Get rid of the really ugly backwards compatability code (that adds
! many, many attributes to the options object) as soon as all the
! modules are changed over.
! o Once the above is done, and we have waited a suitable time, stop
! allowing invalid options in configuration files
o Find a regex expert to come up with *good* patterns for domains,
email addresses, and so forth.
--- 23,27 ----
To Do:
! o Stop allowing invalid options in configuration files
o Find a regex expert to come up with *good* patterns for domains,
email addresses, and so forth.
***************
*** 491,511 ****
o = klass(*args)
self._options[section, o.name] = o
- # A (really ugly) bit of backwards compatability
- # *** This will vanish soon, so do not make use of it in
- # new code ***
- self._oldset(section, o.name, o.value)
- def _oldset(self, section, option, value):
- # A (really ugly) bit of backwards compatability
- # *** This will vanish soon, so do not make use of it in
- # new code ***
- for (oldsect, oldopt), (newsect, newopt) in self.conversion_table.items():
- if (newsect, newopt) == (section, option):
- section = oldsect
- option = oldopt
- setattr(self, option, value)
- old_name = section[0:1].lower() + section[1:] + "_" + option
- setattr(self, old_name, value)
-
def merge_files(self, file_list):
for file in file_list:
--- 487,491 ----
***************
*** 521,537 ****
section = sect
option = opt
- # backward compatibility guff, but only if needed.
- if self.conversion_table:
- if opt[:len(sect) + 1].lower() == sect.lower() + '_':
- opt = opt[len(sect)+1:]
- if self.conversion_table.has_key((sect, opt)):
- section, option = self.conversion_table[sect, opt]
- else:
- section = sect
- option = opt
- else: # no b/w compat needed.
- section = sect
- option = opt
- # end of backward compatibility guff
if not self._options.has_key((section, option)):
print "Invalid option %s in section %s in file %s" % \
--- 501,504 ----
***************
*** 542,549 ****
value = self.convert(section, option, value)
self.set(section, option, value)
- # backward compatibility guff
- if self.conversion_table:
- self._oldset(sect, opt, value)
- # end of backward compatibility guff
# not strictly necessary, but convenient shortcuts to self._options
--- 509,512 ----
***************
*** 603,608 ****
if self.is_valid(sect, opt, val):
self._options[sect, opt].set(val)
- # backwards compatibility stuff
- self._oldset(sect, opt, val)
else:
print "Attempted to set [%s] %s with invalid value %s (%s)" % \
--- 566,569 ----
Index: TestDriver.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/TestDriver.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** TestDriver.py 29 Jan 2003 03:23:34 -0000 1.3
--- TestDriver.py 5 Sep 2003 01:15:28 -0000 1.4
***************
*** 46,50 ****
! def printhist(tag, ham, spam, nbuckets=options.nbuckets):
print
print "-> Ham scores for", tag,
--- 46,50 ----
! def printhist(tag, ham, spam, nbuckets=options["TestDriver", "nbuckets"]):
print
print "-> Ham scores for", tag,
***************
*** 55,59 ****
spam.display(nbuckets)
! if not options.compute_best_cutoffs_from_histograms:
return
if ham.n == 0 or spam.n == 0:
--- 55,59 ----
spam.display(nbuckets)
! if not options["TestDriver", "compute_best_cutoffs_from_histograms"]:
return
if ham.n == 0 or spam.n == 0:
***************
*** 70,76 ****
assert ham.nbuckets == spam.nbuckets
n = ham.nbuckets
! FPW = options.best_cutoff_fp_weight
! FNW = options.best_cutoff_fn_weight
! UNW = options.best_cutoff_unsure_weight
# Get running totals: {h,s}total[i] is # of ham/spam below bucket i
--- 70,76 ----
assert ham.nbuckets == spam.nbuckets
n = ham.nbuckets
! FPW = options["TestDriver", "best_cutoff_fp_weight"]
! FNW = options["TestDriver", "best_cutoff_fn_weight"]
! UNW = options["TestDriver", "best_cutoff_unsure_weight"]
# Get running totals: {h,s}total[i] is # of ham/spam below bucket i
***************
*** 137,142 ****
print
guts = str(msg)
! if options.show_charlimit > 0:
! guts = guts[:options.show_charlimit]
print guts
--- 137,142 ----
print
guts = str(msg)
! if options["TestDriver", "show_charlimit"] > 0:
! guts = guts[:options["TestDriver", "show_charlimit"]]
print guts
***************
*** 181,185 ****
def finishtest(self):
! if options.show_histograms:
printhist("all in this training set:",
self.trained_ham_hist, self.trained_spam_hist)
--- 181,185 ----
def finishtest(self):
! if options["TestDriver", "show_histograms"]:
printhist("all in this training set:",
self.trained_ham_hist, self.trained_spam_hist)
***************
*** 190,195 ****
self.ntimes_finishtest_called += 1
! if options.save_trained_pickles:
! fname = "%s%d.pik" % (options.pickle_basename,
self.ntimes_finishtest_called)
print " saving pickle to", fname
--- 190,195 ----
self.ntimes_finishtest_called += 1
! if options["TestDriver", "save_trained_pickles"]:
! fname = "%s%d.pik" % (options["TestDriver", "pickle_basename"],
self.ntimes_finishtest_called)
print " saving pickle to", fname
***************
*** 199,209 ****
def alldone(self):
! if options.show_histograms:
besthamcut,bestspamcut = printhist("all runs:",
self.global_ham_hist,
self.global_spam_hist)
else:
! besthamcut = options.ham_cutoff
! bestspamcut = options.spam_cutoff
nham = self.global_ham_hist.n
nspam = self.global_spam_hist.n
--- 199,209 ----
def alldone(self):
! if options["TestDriver", "show_histograms"]:
besthamcut,bestspamcut = printhist("all runs:",
self.global_ham_hist,
self.global_spam_hist)
else:
! besthamcut = options["Categorization", "ham_cutoff"]
! bestspamcut = options["Categorization", "spam_cutoff"]
nham = self.global_ham_hist.n
nspam = self.global_spam_hist.n
***************
*** 218,233 ****
print "-> all runs unsure %:", (nun * 1e2 / (nham + nspam))
print "-> all runs cost: $%.2f" % (
! nfp * options.best_cutoff_fp_weight +
! nfn * options.best_cutoff_fn_weight +
! nun * options.best_cutoff_unsure_weight)
# Set back the options for the delayed calculations in self.cc
! options.ham_cutoff = besthamcut
! options.spam_cutoff = bestspamcut
print self.cc
! if options.save_histogram_pickles:
for f, h in (('ham', self.global_ham_hist),
('spam', self.global_spam_hist)):
! fname = "%s_%shist.pik" % (options.pickle_basename, f)
print " saving %s histogram pickle to %s" %(f, fname)
fp = file(fname, 'wb')
--- 218,234 ----
print "-> all runs unsure %:", (nun * 1e2 / (nham + nspam))
print "-> all runs cost: $%.2f" % (
! nfp * options["TestDriver", "best_cutoff_fp_weight"] +
! nfn * options["TestDriver", "best_cutoff_fn_weight"] +
! nun * options["TestDriver", "best_cutoff_unsure_weight"])
# Set back the options for the delayed calculations in self.cc
! options["Categorization", "ham_cutoff"] = besthamcut
! options["Categorization", "spam_cutoff"] = bestspamcut
print self.cc
! if options["TestDriver", "save_histogram_pickles"]:
for f, h in (('ham', self.global_ham_hist),
('spam', self.global_spam_hist)):
! fname = "%s_%shist.pik" % (options["TestDriver",
! "pickle_basename"], f)
print " saving %s histogram pickle to %s" %(f, fname)
fp = file(fname, 'wb')
***************
*** 241,246 ****
local_spam_hist = Hist()
! def new_ham(msg, prob, lo=options.show_ham_lo,
! hi=options.show_ham_hi):
local_ham_hist.add(prob * 100.0)
self.cc.ham(prob)
--- 242,247 ----
local_spam_hist = Hist()
! def new_ham(msg, prob, lo=options["TestDriver", "show_ham_lo"],
! hi=options["TestDriver", "show_ham_hi"]):
local_ham_hist.add(prob * 100.0)
self.cc.ham(prob)
***************
*** 251,256 ****
printmsg(msg, prob, clues)
! def new_spam(msg, prob, lo=options.show_spam_lo,
! hi=options.show_spam_hi):
local_spam_hist.add(prob * 100.0)
self.cc.spam(prob)
--- 252,257 ----
printmsg(msg, prob, clues)
! def new_spam(msg, prob, lo=options["TestDriver", "show_spam_lo"],
! hi=options["TestDriver", "show_spam_hi"]):
local_spam_hist.add(prob * 100.0)
self.cc.spam(prob)
***************
*** 272,279 ****
print "-> unsure %:", t.unsure_rate()
print "-> cost: $%.2f" % (
! t.nham_wrong * options.best_cutoff_fp_weight +
! t.nspam_wrong * options.best_cutoff_fn_weight +
(t.nham_unsure + t.nspam_unsure) *
! options.best_cutoff_unsure_weight)
newfpos = Set(t.false_positives()) - self.falsepos
--- 273,280 ----
print "-> unsure %:", t.unsure_rate()
print "-> cost: $%.2f" % (
! t.nham_wrong * options["TestDriver", "best_cutoff_fp_weight"] +
! t.nspam_wrong * options["TestDriver", "best_cutoff_fn_weight"] +
(t.nham_unsure + t.nspam_unsure) *
! options["TestDriver", "best_cutoff_unsure_weight"])
newfpos = Set(t.false_positives()) - self.falsepos
***************
*** 282,286 ****
if newfpos:
print " new fp:", [e.tag for e in newfpos]
! if not options.show_false_positives:
newfpos = ()
for e in newfpos:
--- 283,287 ----
if newfpos:
print " new fp:", [e.tag for e in newfpos]
! if not options["TestDriver", "show_false_positives"]:
newfpos = ()
for e in newfpos:
***************
*** 294,298 ****
if newfneg:
print " new fn:", [e.tag for e in newfneg]
! if not options.show_false_negatives:
newfneg = ()
for e in newfneg:
--- 295,299 ----
if newfneg:
print " new fn:", [e.tag for e in newfneg]
! if not options["TestDriver", "show_false_negatives"]:
newfneg = ()
for e in newfneg:
***************
*** 306,310 ****
if newunsure:
print " new unsure:", [e.tag for e in newunsure]
! if not options.show_unsure:
newunsure = ()
for e in newunsure:
--- 307,311 ----
if newunsure:
print " new unsure:", [e.tag for e in newunsure]
! if not options["TestDriver", "show_unsure"]:
newunsure = ()
for e in newunsure:
***************
*** 313,317 ****
printmsg(e, prob, clues)
! if options.show_histograms:
printhist("this pair:", local_ham_hist, local_spam_hist)
self.trained_ham_hist += local_ham_hist
--- 314,318 ----
printmsg(e, prob, clues)
! if options["TestDriver", "show_histograms"]:
printhist("this pair:", local_ham_hist, local_spam_hist)
self.trained_ham_hist += local_ham_hist
Index: classifier.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** classifier.py 1 Sep 2003 21:07:32 -0000 1.8
--- classifier.py 5 Sep 2003 01:15:28 -0000 1.9
***************
*** 176,180 ****
return prob
! if options.use_gary_combining:
spamprob = gary_spamprob
--- 176,180 ----
return prob
! if options["Classifier", "use_gary_combining"]:
spamprob = gary_spamprob
***************
*** 262,266 ****
return prob
! if options.use_chi_squared_combining:
spamprob = chi2_spamprob
--- 262,266 ----
return prob
! if options["Classifier", "use_chi_squared_combining"]:
spamprob = chi2_spamprob
***************
*** 313,317 ****
prob = spamratio / (hamratio + spamratio)
! if options.experimental_ham_spam_imbalance_adjustment:
spam2ham = min(nspam / nham, 1.0)
ham2spam = min(nham / nspam, 1.0)
--- 313,317 ----
prob = spamratio / (hamratio + spamratio)
! if options["Classifier", "experimental_ham_spam_imbalance_adjustment"]:
spam2ham = min(nspam / nham, 1.0)
ham2spam = min(nham / nspam, 1.0)
***************
*** 319,324 ****
spam2ham = ham2spam = 1.0
! S = options.unknown_word_strength
! StimesX = S * options.unknown_word_prob
--- 319,324 ----
spam2ham = ham2spam = 1.0
! S = options["Classifier", "unknown_word_strength"]
! StimesX = S * options["Classifier", "unknown_word_prob"]
***************
*** 447,452 ****
def _getclues(self, wordstream):
! mindist = options.minimum_prob_strength
! unknown = options.unknown_word_prob
clues = [] # (distance, prob, word, record) tuples
--- 447,452 ----
def _getclues(self, wordstream):
! mindist = options["Classifier", "minimum_prob_strength"]
! unknown = options["Classifier", "unknown_word_prob"]
clues = [] # (distance, prob, word, record) tuples
***************
*** 464,469 ****
clues.sort()
! if len(clues) > options.max_discriminators:
! del clues[0 : -options.max_discriminators]
# Return (prob, word, record).
return [t[1:] for t in clues]
--- 464,469 ----
clues.sort()
! if len(clues) > options["Classifier", "max_discriminators"]:
! del clues[0 : -options["Classifier", "max_discriminators"]]
# Return (prob, word, record).
return [t[1:] for t in clues]
Index: dbmstorage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/dbmstorage.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** dbmstorage.py 7 Aug 2003 00:05:14 -0000 1.7
--- dbmstorage.py 5 Sep 2003 01:15:28 -0000 1.8
***************
*** 47,51 ****
def open(*args):
! dbm_type = options.dbm_type.lower()
f = open_funcs.get(dbm_type)
if not f:
--- 47,51 ----
def open(*args):
! dbm_type = options["globals", "dbm_type"].lower()
f = open_funcs.get(dbm_type)
if not f:
Index: hammie.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** hammie.py 26 Aug 2003 12:57:31 -0000 1.7
--- hammie.py 5 Sep 2003 01:15:28 -0000 1.8
***************
*** 44,49 ****
for word, prob in clues
if (word[0] == '*' or
! prob <= options.clue_mailheader_cutoff or
! prob >= 1.0 - options.clue_mailheader_cutoff)])
def score(self, msg, evidence=False):
--- 44,51 ----
for word, prob in clues
if (word[0] == '*' or
! prob <= options["Headers",
! "clue_mailheader_cutoff"] or
! prob >= 1.0 - options["Headers",
! "clue_mailheader_cutoff"])])
def score(self, msg, evidence=False):
***************
*** 87,101 ****
if header == None:
! header = options.hammie_header_name
if spam_cutoff == None:
! spam_cutoff = options.spam_cutoff
if ham_cutoff == None:
! ham_cutoff = options.ham_cutoff
if debugheader == None:
! debugheader = options.hammie_debug_header_name
if debug == None:
! debug = options.hammie_debug_header
if train == None:
! train = options.hammie_train_on_filter
msg = mboxutils.get_message(msg)
--- 89,103 ----
if header == None:
! header = options["Headers", "classification_header_name"]
if spam_cutoff == None:
! spam_cutoff = options["Categorization", "spam_cutoff"]
if ham_cutoff == None:
! ham_cutoff = options["Categorization", "ham_cutoff"]
if debugheader == None:
! debugheader = options["Hammie", "debug_header_name"]
if debug == None:
! debug = options["Hammie", "debug_header"]
if train == None:
! train = options["Hammie", "hammie_train_on_filter"]
msg = mboxutils.get_message(msg)
***************
*** 109,123 ****
if prob < ham_cutoff:
is_spam = False
! disp = options.header_ham_string
elif prob > spam_cutoff:
is_spam = True
! disp = options.header_spam_string
else:
is_spam = False
! disp = options.header_unsure_string
if train:
self.train(msg, is_spam, True)
! disp += ("; %."+str(options.header_score_digits)+"f") % prob
! if options.header_score_logarithm:
if prob<=0.005 and prob>0.0:
import math
--- 111,125 ----
if prob < ham_cutoff:
is_spam = False
! disp = options["Headers", "header_ham_string"]
elif prob > spam_cutoff:
is_spam = True
! disp = options["Headers", "header_spam_string"]
else:
is_spam = False
! disp = options["Headers", "header_unsure_string"]
if train:
self.train(msg, is_spam, True)
! disp += ("; %."+str(options["Headers", "header_score_digits"])+"f") % prob
! if options["Headers", "header_score_logarithm"]:
if prob<=0.005 and prob>0.0:
import math
***************
*** 151,159 ****
if add_header:
if is_spam:
! trained = options.header_spam_string
else:
! trained = options.header_ham_string
! del msg[options.hammie_trained_header]
! msg.add_header(options.hammie_trained_header, trained)
def untrain(self, msg, is_spam):
--- 153,161 ----
if add_header:
if is_spam:
! trained = options["Headers", "header_spam_string"]
else:
! trained = options["Headers", "header_ham_string"]
! del msg[options["Headers", "trained_header_name"]]
! msg.add_header(options["Headers", "trained_header_name"], trained)
def untrain(self, msg, is_spam):
***************
*** 181,195 ****
msg = mboxutils.get_message(msg)
! trained = msg.get(options.hammie_trained_header)
if not trained:
return
! del msg[options.hammie_trained_header]
! if trained == options.header_ham_string:
self.untrain_ham(msg)
! elif trained == options.header_spam_string:
self.untrain_spam(msg)
else:
raise ValueError('%s header value unrecognized'
! % options.hammie_trained_header)
def train_ham(self, msg, add_header=False):
--- 183,197 ----
msg = mboxutils.get_message(msg)
! trained = msg.get(options["Headers", "trained_header_name"])
if not trained:
return
! del msg[options["Headers", "trained_header_name"]]
! if trained == options["Headers", "header_ham_string"]:
self.untrain_ham(msg)
! elif trained == options["Headers", "header_spam_string"]:
self.untrain_spam(msg)
else:
raise ValueError('%s header value unrecognized'
! % options["Headers", "trained_header_name"])
def train_ham(self, msg, add_header=False):
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** storage.py 1 Sep 2003 21:07:32 -0000 1.27
--- storage.py 5 Sep 2003 01:15:28 -0000 1.28
***************
*** 101,105 ****
# tempbayes object is reclaimed when load() returns.
! if options.verbose:
print >> sys.stderr, 'Loading state from',self.db_name,'pickle'
--- 101,105 ----
# tempbayes object is reclaimed when load() returns.
! if options["globals", "verbose"]:
print >> sys.stderr, 'Loading state from',self.db_name,'pickle'
***************
*** 119,123 ****
classifier.Classifier.__setstate__(self,
tempbayes.__getstate__())
! if options.verbose:
print >> sys.stderr, ('%s is an existing pickle,'
' with %d ham and %d spam') \
--- 119,123 ----
classifier.Classifier.__setstate__(self,
tempbayes.__getstate__())
! if options["globals", "verbose"]:
print >> sys.stderr, ('%s is an existing pickle,'
' with %d ham and %d spam') \
***************
*** 125,129 ****
else:
# new pickle
! if options.verbose:
print >> sys.stderr, self.db_name,'is a new pickle'
self.wordinfo = {}
--- 125,129 ----
else:
# new pickle
! if options["globals", "verbose"]:
print >> sys.stderr, self.db_name,'is a new pickle'
self.wordinfo = {}
***************
*** 134,138 ****
'''Store self as a pickle'''
! if options.verbose:
print >> sys.stderr, 'Persisting',self.db_name,'as a pickle'
--- 134,138 ----
'''Store self as a pickle'''
! if options["globals", "verbose"]:
print >> sys.stderr, 'Persisting',self.db_name,'as a pickle'
***************
*** 160,164 ****
'''Load state from database'''
! if options.verbose:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
--- 160,164 ----
'''Load state from database'''
! if options["globals", "verbose"]:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
***************
*** 172,176 ****
(self.nspam, self.nham) = t[1:]
! if options.verbose:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
--- 172,176 ----
(self.nspam, self.nham) = t[1:]
! if options["globals", "verbose"]:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
***************
*** 178,182 ****
else:
# new database
! if options.verbose:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
--- 178,182 ----
else:
# new database
! if options["globals", "verbose"]:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
***************
*** 188,192 ****
'''Place state into persistent store'''
! if options.verbose:
print >> sys.stderr, 'Persisting',self.db_name,'state in database'
--- 188,192 ----
'''Place state into persistent store'''
! if options["globals", "verbose"]:
print >> sys.stderr, 'Persisting',self.db_name,'state in database'
***************
*** 424,428 ****
import psycopg
! if options.verbose:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
--- 424,428 ----
import psycopg
! if options["globals", "verbose"]:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
***************
*** 440,444 ****
self.nspam = row["nspam"]
self.nham = row["nham"]
! if options.verbose:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
--- 440,444 ----
self.nspam = row["nspam"]
self.nham = row["nham"]
! if options["globals", "verbose"]:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
***************
*** 446,450 ****
else:
# new database
! if options.verbose:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
--- 446,450 ----
else:
# new database
! if options["globals", "verbose"]:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
***************
*** 495,499 ****
import MySQLdb
! if options.verbose:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
--- 495,499 ----
import MySQLdb
! if options["globals", "verbose"]:
print >> sys.stderr, 'Loading state from',self.db_name,'database'
***************
*** 512,516 ****
self.nspam = int(row[1])
self.nham = int(row[2])
! if options.verbose:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
--- 512,516 ----
self.nspam = int(row[1])
self.nham = int(row[2])
! if options["globals", "verbose"]:
print >> sys.stderr, ('%s is an existing database,'
' with %d spam and %d ham') \
***************
*** 518,522 ****
else:
# new database
! if options.verbose:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
--- 518,522 ----
else:
# new database
! if options["globals", "verbose"]:
print >> sys.stderr, self.db_name,'is a new database'
self.nspam = 0
***************
*** 555,559 ****
'''Train the database with the message'''
! if options.verbose:
print >> sys.stderr, 'training with',message.key()
--- 555,559 ----
'''Train the database with the message'''
! if options["globals", "verbose"]:
print >> sys.stderr, 'training with',message.key()
***************
*** 569,573 ****
'''Untrain the database with the message'''
! if options.verbose:
print >> sys.stderr, 'untraining with',message.key()
--- 569,573 ----
'''Untrain the database with the message'''
! if options["globals", "verbose"]:
print >> sys.stderr, 'untraining with',message.key()
Index: tokenizer.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/tokenizer.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** tokenizer.py 28 Jun 2003 01:51:57 -0000 1.13
--- tokenizer.py 5 Sep 2003 01:15:28 -0000 1.14
***************
*** 657,661 ****
yield "fname piece:" + piece
! def tokenize_word(word, _len=len, maxword=options.skip_max_word_size):
n = _len(word)
# Make sure this range matches in tokenize().
--- 657,662 ----
yield "fname piece:" + piece
! def tokenize_word(word, _len=len, maxword=options["Tokenizer",
! "skip_max_word_size"]):
n = _len(word)
# Make sure this range matches in tokenize().
***************
*** 680,684 ****
# XXX Figure out why, and/or see if some other way of summarizing
# XXX this info has greater benefit.
! if options.generate_long_skips:
yield "skip:%c %d" % (word[0], n // 10 * 10)
if has_highbit_char(word):
--- 681,685 ----
# XXX Figure out why, and/or see if some other way of summarizing
# XXX this info has greater benefit.
! if options["Tokenizer", "generate_long_skips"]:
yield "skip:%c %d" % (word[0], n // 10 * 10)
if has_highbit_char(word):
***************
*** 1070,1076 ****
def __init__(self):
! if options.basic_header_tokenize:
self.basic_skip = [re.compile(s)
! for s in options.basic_header_skip]
def get_message(self, obj):
--- 1071,1078 ----
def __init__(self):
! if options["Tokenizer", "basic_header_tokenize"]:
self.basic_skip = [re.compile(s)
! for s in options["Tokenizer",
! "basic_header_skip"]]
def get_message(self, obj):
***************
*** 1115,1119 ****
# the best discriminators.
# (Not just Date, but Received and X-From_.)
! if options.basic_header_tokenize:
for k, v in msg.items():
k = k.lower()
--- 1117,1121 ----
# the best discriminators.
# (Not just Date, but Received and X-From_.)
! if options["Tokenizer", "basic_header_tokenize"]:
for k, v in msg.items():
k = k.lower()
***************
*** 1126,1130 ****
for t in tokenize_word(w):
yield "%s:%s" % (k, t)
! if options.basic_header_tokenize_only:
return
--- 1128,1132 ----
for t in tokenize_word(w):
yield "%s:%s" % (k, t)
! if options["Tokenizer", "basic_header_tokenize_only"]:
return
***************
*** 1157,1161 ****
# To:, Cc: # These can help, if your ham and spam are sourced
# # from the same location. If not, they'll be horrible.
! for field in options.address_headers:
addrlist = msg.get_all(field, [])
if not addrlist:
--- 1159,1163 ----
# To:, Cc: # These can help, if your ham and spam are sourced
# # from the same location. If not, they'll be horrible.
! for field in options["Tokenizer", "address_headers"]:
addrlist = msg.get_all(field, [])
if not addrlist:
***************
*** 1199,1203 ****
# eliminates the bad case where the message was sent to a single
# individual.
! if options.summarize_email_prefixes:
all_addrs = []
addresses = msg.get_all('to', []) + msg.get_all('cc', [])
--- 1201,1205 ----
# eliminates the bad case where the message was sent to a single
# individual.
! if options["Tokenizer", "summarize_email_prefixes"]:
all_addrs = []
addresses = msg.get_all('to', []) + msg.get_all('cc', [])
***************
*** 1226,1230 ****
# , ,
# , ,
! if options.summarize_email_suffixes:
all_addrs = []
addresses = msg.get_all('to', []) + msg.get_all('cc', [])
--- 1228,1232 ----
# , ,
# , ,
! if options["Tokenizer", "summarize_email_suffixes"]:
all_addrs = []
addresses = msg.get_all('to', []) + msg.get_all('cc', [])
***************
*** 1273,1277 ****
# Received:
# Neil Schemenauer reports good results from this.
! if options.mine_received_headers:
for header in msg.get_all("received", ()):
for pat, breakdown in [(received_host_re, breakdown_host),
--- 1275,1279 ----
# Received:
# Neil Schemenauer reports good results from this.
! if options["Tokenizer", "mine_received_headers"]:
for header in msg.get_all("received", ()):
for pat, breakdown in [(received_host_re, breakdown_host),
***************
*** 1283,1287 ****
# Date:
! if options.generate_time_buckets:
for header in msg.get_all("date", ()):
mat = self.date_hms_re.search(header)
--- 1285,1289 ----
# Date:
! if options["Tokenizer", "generate_time_buckets"]:
for header in msg.get_all("date", ()):
mat = self.date_hms_re.search(header)
***************
*** 1293,1297 ****
yield 'time:%02d:%d' % (h, bucket)
! if options.extract_dow:
for header in msg.get_all("date", ()):
# extract the day of the week
--- 1295,1299 ----
yield 'time:%02d:%d' % (h, bucket)
! if options["Tokenizer", "extract_dow"]:
for header in msg.get_all("date", ()):
# extract the day of the week
***************
*** 1324,1328 ****
# X-Complaints-To a strong ham clue.
x2n = {}
! if options.count_all_header_lines:
for x in msg.keys():
x2n[x] = x2n.get(x, 0) + 1
--- 1326,1330 ----
# X-Complaints-To a strong ham clue.
x2n = {}
! if options["Tokenizer", "count_all_header_lines"]:
for x in msg.keys():
x2n[x] = x2n.get(x, 0) + 1
***************
*** 1343,1347 ****
yield "noheader:" + k
! def tokenize_body(self, msg, maxword=options.skip_max_word_size):
"""Generate a stream of tokens from an email Message.
--- 1345,1350 ----
yield "noheader:" + k
! def tokenize_body(self, msg, maxword=options["Tokenizer",
! "skip_max_word_size"]):
"""Generate a stream of tokens from an email Message.
***************
*** 1350,1354 ****
"""
! if options.check_octets:
# Find, decode application/octet-stream parts of the body,
# tokenizing the first few characters of each chunk.
--- 1353,1357 ----
"""
! if options["Tokenizer", "check_octets"]:
# Find, decode application/octet-stream parts of the body,
# tokenizing the first few characters of each chunk.
***************
*** 1364,1368 ****
continue
! yield "octet:%s" % text[:options.octet_prefix_size]
# Find, decode (base64, qp), and tokenize textual parts of the body.
--- 1367,1372 ----
continue
! yield "octet:%s" % text[:options["Tokenizer",
! "octet_prefix_size"]]
# Find, decode (base64, qp), and tokenize textual parts of the body.
***************
*** 1388,1392 ****
text = text.lower()
! if options.replace_nonascii_chars:
# Replace high-bit chars and control chars with '?'.
text = text.translate(non_ascii_translate_tab)
--- 1392,1396 ----
text = text.lower()
! if options["Tokenizer", "replace_nonascii_chars"]:
# Replace high-bit chars and control chars with '?'.
text = text.translate(non_ascii_translate_tab)
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:16:52 2003
Subject: [Spambayes-checkins] spambayes/testtools pop3proxytest.py,1.1,NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes/testtools
In directory sc8-pr-cvs1:/tmp/cvs-serv14316/testtools
Removed Files:
pop3proxytest.py
Log Message:
Crap! We can't use "sb-" as a prefix, because then we can't import the scripts.
I guess that all the importable code could be moved into modules, but that seems
like a huge hassle. Let's use "sb_" as a prefix instead.
Apologies for cluttering the attic...sigh.
--- pop3proxytest.py DELETED ---
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:16:57 2003
Subject: [Spambayes-checkins] spambayes/spambayes/test test_sb-server.py,
NONE, 1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv14316/spambayes/test
Added Files:
test_sb-server.py
Log Message:
Crap! We can't use "sb-" as a prefix, because then we can't import the scripts.
I guess that all the importable code could be moved into modules, but that seems
like a huge hassle. Let's use "sb_" as a prefix instead.
Apologies for cluttering the attic...sigh.
--- NEW FILE: test_sb-server.py ---
#! /usr/bin/env python
"""Test the pop3proxy is working correctly.
When using the -z command line option, carries out a test that the
POP3 proxy can be connected to, that incoming mail is classified,
that pipelining is removed from the CAPA[bility] query, and that the
web ui is present.
The -t option runs a fake POP3 server on port 8810. This is the
same server that the -z option uses, and may be separately run for
other testing purposes.
Usage:
pop3proxytest.py [options]
options:
-z : Runs a self-test and exits.
-t : Runs a fake POP3 server on port 8110 (for testing).
-h : Displays this help message.
"""
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Richie Hindle "
__credits__ = "All the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
# This code originally formed a part of pop3proxy.py. If you are examining
# the history of this file, you may need to go back to there.
todo = """
Web training interface:
o Functional tests.
"""
# One example of spam and one of ham - both are used to train, and are
# then classified. Not a good test of the classifier, but a perfectly
# good test of the POP3 proxy. The bodies of these came from the
# spambayes project, and Richie added the headers because the
# originals had no headers.
spam1 = """From: friend@public.com
Subject: Make money fast
Hello tim_chandler , Want to save money ?
Now is a good time to consider refinancing. Rates are low so you can cut
your current payments and save money.
http://64.251.22.101/interest/index%38%30%300%2E%68t%6D
Take off list on site [s5]
"""
good1 = """From: chris@example.com
Subject: ZPT and DTML
Jean Jordaan wrote:
> 'Fraid so ;> It contains a vintage dtml-calendar tag.
> http://www.zope.org/Members/teyc/CalendarTag
>
> Hmm I think I see what you mean: one needn't manually pass on the
> namespace to a ZPT?
Yeah, Page Templates are a bit more clever, sadly, DTML methods aren't :-(
Chris
"""
import asyncore
import socket
import operator
import re
import getopt
# a bit of a hack to help those without spambayes on their
# Python path - stolen from timtest.py
import sys
import os
sys.path.insert(-1, os.getcwd())
sys.path.insert(-1, os.path.dirname(os.getcwd()))
from spambayes import Dibbler
from spambayes import tokenizer
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ProxyUI import ProxyUserInterface
from pop3proxy import BayesProxyListener
from pop3proxy import state, _recreateState
from spambayes.Options import options
# HEADER_EXAMPLE is the longest possible header - the length of this one
# is added to the size of each message.
HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \
options["Hammie", "header_name"]
class TestListener(Dibbler.Listener):
"""Listener for TestPOP3Server. Works on port 8110, to co-exist
with real POP3 servers."""
def __init__(self, socketMap=asyncore.socket_map):
Dibbler.Listener.__init__(self, 8110, TestPOP3Server,
(socketMap,), socketMap=socketMap)
class TestPOP3Server(Dibbler.BrighterAsyncChat):
"""Minimal POP3 server, for testing purposes. Doesn't support
UIDL. USER, PASS, APOP, DELE and RSET simply return "+OK"
without doing anything. Also understands the 'KILL' command, to
kill it. The mail content is the example messages above.
"""
def __init__(self, clientSocket, socketMap):
# Grumble: asynchat.__init__ doesn't take a 'map' argument,
# hence the two-stage construction.
Dibbler.BrighterAsyncChat.__init__(self)
Dibbler.BrighterAsyncChat.set_socket(self, clientSocket, socketMap)
self.maildrop = [spam1, good1]
self.set_terminator('\r\n')
self.okCommands = ['USER', 'PASS', 'APOP', 'NOOP',
'DELE', 'RSET', 'QUIT', 'KILL']
self.handlers = {'CAPA': self.onCapa,
'STAT': self.onStat,
'LIST': self.onList,
'RETR': self.onRetr,
'TOP': self.onTop}
self.push("+OK ready\r\n")
self.request = ''
def collect_incoming_data(self, data):
"""Asynchat override."""
self.request = self.request + data
def found_terminator(self):
"""Asynchat override."""
if ' ' in self.request:
command, args = self.request.split(None, 1)
else:
command, args = self.request, ''
command = command.upper()
if command in self.okCommands:
self.push("+OK (we hope)\r\n")
if command == 'QUIT':
self.close_when_done()
if command == 'KILL':
self.socket.shutdown(2)
self.close()
raise SystemExit
else:
handler = self.handlers.get(command, self.onUnknown)
self.push(handler(command, args)) # Or push_slowly for testing
self.request = ''
def push_slowly(self, response):
"""Useful for testing."""
for c in response:
self.push(c)
time.sleep(0.02)
def onCapa(self, command, args):
"""POP3 CAPA command. This lies about supporting pipelining for
test purposes - the POP3 proxy *doesn't* support pipelining, and
we test that it correctly filters out that capability from the
proxied capability list."""
lines = ["+OK Capability list follows",
"PIPELINING",
"TOP",
".",
""]
return '\r\n'.join(lines)
def onStat(self, command, args):
"""POP3 STAT command."""
maildropSize = reduce(operator.add, map(len, self.maildrop))
maildropSize += len(self.maildrop) * len(HEADER_EXAMPLE)
return "+OK %d %d\r\n" % (len(self.maildrop), maildropSize)
def onList(self, command, args):
"""POP3 LIST command, with optional message number argument."""
if args:
try:
number = int(args)
except ValueError:
number = -1
if 0 < number <= len(self.maildrop):
return "+OK %d\r\n" % len(self.maildrop[number-1])
else:
return "-ERR no such message\r\n"
else:
returnLines = ["+OK"]
for messageIndex in range(len(self.maildrop)):
size = len(self.maildrop[messageIndex])
returnLines.append("%d %d" % (messageIndex + 1, size))
returnLines.append(".")
return '\r\n'.join(returnLines) + '\r\n'
def _getMessage(self, number, maxLines):
"""Implements the POP3 RETR and TOP commands."""
if 0 < number <= len(self.maildrop):
message = self.maildrop[number-1]
headers, body = message.split('\n\n', 1)
bodyLines = body.split('\n')[:maxLines]
message = headers + '\r\n\r\n' + '\n'.join(bodyLines)
return "+OK\r\n%s\r\n.\r\n" % message
else:
return "-ERR no such message\r\n"
def onRetr(self, command, args):
"""POP3 RETR command."""
try:
number = int(args)
except ValueError:
number = -1
return self._getMessage(number, 12345)
def onTop(self, command, args):
"""POP3 RETR command."""
try:
number, lines = map(int, args.split())
except ValueError:
number, lines = -1, -1
return self._getMessage(number, lines)
def onUnknown(self, command, args):
"""Unknown POP3 command."""
return "-ERR Unknown command: %s\r\n" % repr(command)
def test():
"""Runs a self-test using TestPOP3Server, a minimal POP3 server
that serves the example emails above.
"""
# Run a proxy and a test server in separate threads with separate
# asyncore environments.
import threading
state.isTest = True
testServerReady = threading.Event()
def runTestServer():
testSocketMap = {}
TestListener(socketMap=testSocketMap)
testServerReady.set()
asyncore.loop(map=testSocketMap)
proxyReady = threading.Event()
def runUIAndProxy():
httpServer = UserInterfaceServer(8881)
proxyUI = ProxyUserInterface(state, _recreateState)
httpServer.register(proxyUI)
BayesProxyListener('localhost', 8110, ('', 8111))
state.bayes.learn(tokenizer.tokenize(spam1), True)
state.bayes.learn(tokenizer.tokenize(good1), False)
proxyReady.set()
Dibbler.run()
threading.Thread(target=runTestServer).start()
testServerReady.wait()
threading.Thread(target=runUIAndProxy).start()
proxyReady.wait()
# Connect to the proxy and the test server.
proxy = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
proxy.connect(('localhost', 8111))
response = proxy.recv(100)
assert response == "+OK ready\r\n"
pop3Server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
pop3Server.connect(('localhost', 8110))
response = pop3Server.recv(100)
assert response == "+OK ready\r\n"
# Verify that the test server claims to support pipelining.
pop3Server.send("capa\r\n")
response = pop3Server.recv(1000)
assert response.find("PIPELINING") >= 0
# Ask for the capabilities via the proxy, and verify that the proxy
# is filtering out the PIPELINING capability.
proxy.send("capa\r\n")
response = proxy.recv(1000)
assert response.find("PIPELINING") == -1
# Stat the mailbox to get the number of messages.
proxy.send("stat\r\n")
response = proxy.recv(100)
count, totalSize = map(int, response.split()[1:3])
assert count == 2
# Loop through the messages ensuring that they have judgement
# headers.
for i in range(1, count+1):
response = ""
proxy.send("retr %d\r\n" % i)
while response.find('\n.\r\n') == -1:
response = response + proxy.recv(1000)
assert response.find(options["Hammie", "header_name"]) >= 0
# Smoke-test the HTML UI.
httpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
httpServer.connect(('localhost', 8881))
httpServer.sendall("get / HTTP/1.0\r\n\r\n")
response = ''
while 1:
packet = httpServer.recv(1000)
if not packet: break
response += packet
assert re.search(r"(?s).*Spambayes proxy.*", response)
# Kill the proxy and the test server.
proxy.sendall("kill\r\n")
proxy.recv(100)
pop3Server.sendall("kill\r\n")
pop3Server.recv(100)
def run():
# Read the arguments.
try:
opts, args = getopt.getopt(sys.argv[1:], 'htz')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
runSelfTest = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-t':
state.isTest = True
state.runTestServer = True
elif opt == '-z':
state.isTest = True
runSelfTest = True
state.createWorkers()
if runSelfTest:
print "\nRunning self-test...\n"
state.buildServerStrings()
test()
print "Self-test passed." # ...else it would have asserted.
elif state.runTestServer:
print "Running a test POP3 server on port 8110..."
TestListener()
asyncore.loop()
else:
print >>sys.stderr, __doc__
if __name__ == '__main__':
run()
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:16:48 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:19:37 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_client.py, NONE,
1.1 sb_dbexpimp.py, NONE, 1.1 sb_filter.py, NONE,
1.1 sb_imapfilter.py, NONE, 1.1 sb_mailsort.py, NONE,
1.1 sb_mboxtrain.py, NONE, 1.1 sb_notesfilter.py, NONE,
1.1 sb_pop3dnd.py, NONE, 1.1 sb_server.py, NONE,
1.1 sb_smtpproxy.py, NONE, 1.1 sb_unheader.py, NONE,
1.1 sb_upload.py, NONE, 1.1 sb_xmlrpcserver.py, NONE,
1.1 sb-client.py, 1.1, NONE sb-dbexpimp.py, 1.1,
NONE sb-filter.py, 1.1, NONE sb-imapfilter.py, 1.1,
NONE sb-mailsort.py, 1.1, NONE sb-mboxtrain.py, 1.1,
NONE sb-notesfilter.py, 1.1, NONE sb-pop3dnd.py, 1.1,
NONE sb-server.py, 1.1, NONE sb-smtpproxy.py, 1.1,
NONE sb-unheader.py, 1.1, NONE sb-upload.py, 1.1,
NONE sb-xmlrpcserver.py, 1.1, NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv14316/scripts
Added Files:
sb_client.py sb_dbexpimp.py sb_filter.py sb_imapfilter.py
sb_mailsort.py sb_mboxtrain.py sb_notesfilter.py sb_pop3dnd.py
sb_server.py sb_smtpproxy.py sb_unheader.py sb_upload.py
sb_xmlrpcserver.py
Removed Files:
sb-client.py sb-dbexpimp.py sb-filter.py sb-imapfilter.py
sb-mailsort.py sb-mboxtrain.py sb-notesfilter.py sb-pop3dnd.py
sb-server.py sb-smtpproxy.py sb-unheader.py sb-upload.py
sb-xmlrpcserver.py
Log Message:
Crap! We can't use "sb-" as a prefix, because then we can't import the scripts.
I guess that all the importable code could be moved into modules, but that seems
like a huge hassle. Let's use "sb_" as a prefix instead.
Apologies for cluttering the attic...sigh.
--- NEW FILE: sb_client.py ---
#! /usr/bin/env python
"""A client for hammiesrv.
Just feed it your mail on stdin, and it spits out the same message
with the spambayes score in a new X-Spambayes-Disposition header.
"""
import xmlrpclib
import sys
RPCBASE="http://localhost:65000"
def main():
msg = sys.stdin.read()
try:
x = xmlrpclib.ServerProxy(RPCBASE)
m = xmlrpclib.Binary(msg)
out = x.filter(m)
print out.data
except:
if __debug__:
import traceback
traceback.print_exc()
print msg
if __name__ == "__main__":
main()
--- NEW FILE: sb_dbexpimp.py ---
#! /usr/bin/env python
"""dbExpImp.py - Bayes database export/import
Classes:
Abstract:
This utility has the primary function of exporting and importing
a spambayes database into/from a flat file. This is useful in a number
of scenarios.
Platform portability of database - flat files can be exported and
imported across platforms (winduhs and linux, for example)
Database implementation changes - databases can survive database
implementation upgrades or new database implementations. For example,
if a dbm implementation changes between python x.y and python x.y+1...
Database reorganization - an export followed by an import reorgs an
existing database, improving performance, at least in
some database implementations
Database sharing - it is possible to distribute particular databases
for research purposes, database sharing purposes, or for new users to
have a 'seed' database to start with.
Database merging - multiple databases can be merged into one quite easily
by simply not specifying -n on an import. This will add the two database
nham and nspams together (assuming the two databases do not share corpora)
and for wordinfo conflicts, will add spamcount and hamcount together.
Spambayes software release migration - an export can be executed before
a release upgrade, as part of the installation script. Then, after the
new software is installed, an import can be executed, which will
effectively preserve existing training. This eliminates the need for
retraining every time a release is installed.
Others? I'm sure I haven't thought of everything...
Usage:
dbExpImp [options]
options:
-e : export
-i : import
-v : verbose mode (some additional diagnostic messages)
-f: FN : flat file to export to or import from
-d: FN : name of pickled database file to use
-D: FN : name of dbm database file to use
-m : merge import into an existing database file. This is
meaningful only for import. If omitted, a new database
file will be created. If specified, the imported
wordinfo will be merged into an existing database.
Run dbExpImp -h for more information.
-h : help
Examples:
Export pickled mybayes.db into mybayes.db.export as a csv flat file
dbExpImp -e -d mybayes.db -f mybayes.db.export
Import mybayes.eb.export into a new DBM mybayes.db
dbExpImp -i -D mybayes.db -f mybayes.db.export
Export, then import (reorganize) new pickled mybayes.db
dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export
Convert a bayes database from pickle to DBM
dbExpImp -e -d abayes.db -f abayes.export
dbExpImp -i -D abayes.db -f abayes.export
Create a new database (newbayes.db) from two
databases (abayes.db, bbayes.db)
dbExpImp -e -d abayes.db -f abayes.export
dbExpImp -e -d bbayes.db -f bbayes.export
dbExpImp -i -d newbayes.db -f abayes.export
dbExpImp -i -m -d newbayes.db -f bbayes.export
To Do:
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tim Stone "
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import spambayes.storage
from spambayes.Options import options
import sys, os, getopt, errno, re
import urllib
def runExport(dbFN, useDBM, outFN):
if useDBM:
bayes = spambayes.storage.DBDictClassifier(dbFN)
words = bayes.db.keys()
words.remove(bayes.statekey)
else:
bayes = spambayes.storage.PickledClassifier(dbFN)
words = bayes.wordinfo.keys()
try:
fp = open(outFN, 'w')
except IOError, e:
if e.errno != errno.ENOENT:
raise
nham = bayes.nham;
nspam = bayes.nspam;
print "Exporting database %s to file %s" % (dbFN, outFN)
print "Database has %s ham, %s spam, and %s words" \
% (nham, nspam, len(words))
fp.write("%s,%s,\n" % (nham, nspam))
for word in words:
wi = bayes._wordinfoget(word)
hamcount = wi.hamcount
spamcount = wi.spamcount
word = urllib.quote(word)
fp.write("%s`%s`%s`\n" % (word, hamcount, spamcount))
fp.close()
def runImport(dbFN, useDBM, newDBM, inFN):
if newDBM:
try:
os.unlink(dbFN)
except OSError, e:
if e.errno != 2: # errno.
raise
try:
os.unlink(dbFN+".dat")
except OSError, e:
if e.errno != 2: # errno.
raise
try:
os.unlink(dbFN+".dir")
except OSError, e:
if e.errno != 2: # errno.
raise
if useDBM:
bayes = spambayes.storage.DBDictClassifier(dbFN)
else:
bayes = spambayes.storage.PickledClassifier(dbFN)
try:
fp = open(inFN, 'r')
except IOError, e:
if e.errno != errno.ENOENT:
raise
nline = fp.readline()
(nham, nspam, junk) = re.split(',', nline)
if newDBM:
bayes.nham = int(nham)
bayes.nspam = int(nspam)
else:
bayes.nham += int(nham)
bayes.nspam += int(nspam)
if newDBM:
impType = "Importing"
else:
impType = "Merging"
print "%s database %s using file %s" % (impType, dbFN, inFN)
lines = fp.readlines()
for line in lines:
(word, hamcount, spamcount, junk) = re.split('`', line)
word = urllib.unquote(word)
try:
wi = bayes.wordinfo[word]
except KeyError:
wi = bayes.WordInfoClass()
wi.hamcount += int(hamcount)
wi.spamcount += int(spamcount)
bayes._wordinfoset(word, wi)
fp.close()
print "Storing database, please be patient. Even moderately large"
print "databases may take a very long time to store."
bayes.store()
print "Finished storing database"
if useDBM:
words = bayes.db.keys()
words.remove(bayes.statekey)
else:
words = bayes.wordinfo.keys()
print "Database has %s ham, %s spam, and %s words" \
% (bayes.nham, bayes.nspam, len(words))
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], 'iehmvd:D:f:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
usePickle = False
useDBM = False
newDBM = True
dbFN = None
flatFN = None
exp = False
imp = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
dbFN = arg
elif opt == '-D':
useDBM = True
dbFN = arg
elif opt == '-f':
flatFN = arg
elif opt == '-e':
exp = True
elif opt == '-i':
imp = True
elif opt == '-m':
newDBM = False
elif opt == '-v':
options["globals", "verbose"] = True
if (dbFN and flatFN):
if exp:
runExport(dbFN, useDBM, flatFN)
if imp:
runImport(dbFN, useDBM, newDBM, flatFN)
else:
print >>sys.stderr, __doc__
--- NEW FILE: sb_filter.py ---
#!/usr/bin/env python
## A hammie front-end to make the simple stuff simple.
##
##
## The intent is to call this from procmail and its ilk like so:
##
## :0 fw
## | hammiefilter.py
##
## Then, you can set up your MUA to pipe ham and spam to it, one at a
## time, by calling it with either the -g or -s options, respectively.
##
## Author: Neale Pickett
##
"""Usage: %(program)s [OPTION]...
[OPTION] is one of:
-h
show usage and exit
-x
show some usage examples and exit
-d DBFILE
use database in DBFILE
-D PICKLEFILE
use pickle (instead of database) in PICKLEFILE
-n
create a new database
*+ -f
filter (default if no processing options are given)
*+ -t
[EXPERIMENTAL] filter and train based on the result (you must
make sure to untrain all mistakes later)
* -g
[EXPERIMENTAL] (re)train as a good (ham) message
* -s
[EXPERIMENTAL] (re)train as a bad (spam) message
* -G
[EXPERIMENTAL] untrain ham (only use if you've already trained
this message)
* -S
[EXPERIMENTAL] untrain spam (only use if you've already trained
this message)
All options marked with '*' operate on stdin. Only those processing options
marked with '+' send a modified message to stdout.
"""
import os
import sys
import getopt
from spambayes import hammie, Options, mboxutils
# See Options.py for explanations of these properties
program = sys.argv[0]
example_doc = """_Examples_
filter a message on disk:
%(program)s < message
(re)train a message as ham:
%(program)s -g < message
(re)train a message as spam:
%(program)s -s < message
procmail recipe to filter and train in one step:
:0 fw
| %(program)s -t
mutt configuration. This binds the 'H' key to retrain the message as
ham, and prompt for a folder to move it to. The 'S' key retrains as
spam, and moves to a 'spam' folder.
XXX: add this
"""
def examples():
print example_doc % globals()
sys.exit(0)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
class HammieFilter(object):
def __init__(self):
options = Options.options
# This is a bit of a hack to counter the default for
# persistent_storage_file changing from ~/.hammiedb to hammie.db
# This will work unless a user:
# * had hammie.db as their value for persistent_storage_file, and
# * their config file was loaded by Options.py.
if options["Storage", "persistent_storage_file"] == \
options.default("Storage", "persistent_storage_file"):
options["Storage", "persistent_storage_file"] = \
"~/.hammiedb"
options.merge_files(['/etc/hammierc',
os.path.expanduser('~/.hammierc')])
self.dbname = options["Storage", "persistent_storage_file"]
self.dbname = os.path.expanduser(self.dbname)
self.usedb = options["Storage", "persistent_use_database"]
def newdb(self):
h = hammie.open(self.dbname, self.usedb, 'n')
h.store()
print >> sys.stderr, "Created new database in", self.dbname
def filter(self, msg):
h = hammie.open(self.dbname, self.usedb, 'r')
return h.filter(msg)
def filter_train(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
return h.filter(msg, train=True)
def train_ham(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.train_ham(msg, True)
h.store()
def train_spam(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.train_spam(msg, True)
h.store()
def untrain_ham(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.untrain_ham(msg)
h.store()
def untrain_spam(self, msg):
h = hammie.open(self.dbname, self.usedb, 'c')
h.untrain_spam(msg)
h.store()
def main():
h = HammieFilter()
actions = []
opts, args = getopt.getopt(sys.argv[1:], 'hxd:D:nfgstGS',
['help', 'examples'])
for opt, arg in opts:
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-x', '--examples'):
examples()
elif opt == '-d':
h.usedb = True
h.dbname = arg
elif opt == '-D':
h.usedb = False
h.dbname = arg
elif opt == '-f':
actions.append(h.filter)
elif opt == '-g':
actions.append(h.train_ham)
elif opt == '-s':
actions.append(h.train_spam)
elif opt == '-t':
actions.append(h.filter_train)
elif opt == '-G':
actions.append(h.untrain_ham)
elif opt == '-S':
actions.append(h.untrain_spam)
elif opt == "-n":
h.newdb()
sys.exit(0)
if actions == []:
actions = [h.filter]
msg = mboxutils.get_message(sys.stdin)
for action in actions:
action(msg)
sys.stdout.write(msg.as_string(unixfrom=(msg.get_unixfrom() is not None)))
if __name__ == "__main__":
main()
--- NEW FILE: sb_imapfilter.py ---
#!/usr/bin/env python
"""An IMAP filter. An IMAP message box is scanned and all non-scored
messages are scored and (where necessary) filtered.
The original filter design owed much to isbg by Roger Binns
(http://www.rogerbinns.com/isbg).
Usage:
imapfilter [options]
note: option values with spaces in them must be enclosed
in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-t : train contents of spam folder and ham folder
-c : classify inbox
-h : help
-v : verbose mode
-p : security option to prompt for imap password,
rather than look in options["imap", "password"]
-e y/n : expunge/purge messages on exit (y) or not (n)
-i debuglvl : a somewhat mysterious imaplib debugging level
-l minutes : period of time between filtering operations
-b : Launch a web browser showing the user interface.
Examples:
Classify inbox, with dbm database
imapfilter -c -D bayes.db
Train Spam and Ham, then classify inbox, with dbm database
imapfilter -t -c -D bayes.db
Train Spam and Ham only, with pickled database
imapfilter -t -d bayes.db
Warnings:
o This is alpha software! The filter is currently being developed and
tested. We do *not* recommend using it on a production system unless
you are confident that you can get your mail back if you lose it. On
the other hand, we do recommend that you test it for us and let us
know if anything does go wrong.
o By default, the filter does *not* delete, modify or move any of your
mail. Due to quirks in how imap works, new versions of your mail are
modified and placed in new folders, but the originals are still
available. These are flagged with the /Deleted flag so that you know
that they can be removed. Your mailer may not show these messages
by default, but there should be an option to do so. *However*, if
your mailer automatically purges/expunges (i.e. permanently deletes)
mail flagged as such, *or* if you set the imap_expunge option to
True, then this mail will be irretrievably lost.
To Do:
o IMAPMessage and IMAPFolder currently carry out very simple checks
of responses received from IMAP commands, but if the response is not
"OK", then the filter terminates. Handling of these errors could be
much nicer.
o IMAP over SSL is untested.
o Develop a test script, like spambayes/test/test_pop3proxy.py that
runs through some tests (perhaps with a *real* imap server, rather
than a dummy one). This would make it easier to carry out the tests
against each server whenever a change is made.
o IMAP supports authentication via other methods than the plain-text
password method that we are using at the moment. Neither of the
servers I have access to offer any alternative method, however. If
someone's does, then it would be nice to offer this.
o Usernames should be able to be literals as well as quoted strings.
This might help if the username/password has special characters like
accented characters.
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer , Tim Stone"
__credits__ = "All the Spambayes folk."
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import socket
import os
import re
import time
import sys
import getopt
import types
import email
import email.Parser
from getpass import getpass
from email.Utils import parsedate
from spambayes.Options import options
from spambayes import tokenizer, storage, message, Dibbler
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ImapUI import IMAPUserInterface
from spambayes.Version import get_version_string
from imaplib import IMAP4
from imaplib import Time2Internaldate
try:
if options["imap", "use_ssl"]:
from imaplib import IMAP_SSL as BaseIMAP
else:
from imaplib import IMAP4 as BaseIMAP
except ImportError:
from imaplib import IMAP4 as BaseIMAP
# global IMAPlib object
global imap
imap = None
# A flag can have any character in the ascii range 32-126
# except for (){ %*"\
FLAG_CHARS = ""
for i in range(32, 127):
if not chr(i) in ['(', ')', '{', ' ', '%', '*', '"', '\\']:
FLAG_CHARS += chr(i)
FLAG = r"\\?[" + re.escape(FLAG_CHARS) + r"]+"
# The empty flag set "()" doesn't match, so that extract returns
# data["FLAGS"] == None
FLAGS_RE = re.compile(r"(FLAGS) (\((" + FLAG + r" )*(" + FLAG + r")\))")
INTERNALDATE_RE = re.compile(r"(INTERNALDATE) (\"\d{1,2}\-[A-Za-z]{3,3}\-" +
r"\d{2,4} \d{2,2}\:\d{2,2}\:\d{2,2} " +
r"[\+\-]\d{4,4}\")")
RFC822_RE = re.compile(r"(RFC822) (\{[\d]+\})")
RFC822_HEADER_RE = re.compile(r"(RFC822.HEADER) (\{[\d]+\})")
UID_RE = re.compile(r"(UID) ([\d]+)")
FETCH_RESPONSE_RE = re.compile(r"([0-9]+) \(([" + \
re.escape(FLAG_CHARS) + r"\"\{\}\(\)\\ ]*)\)?")
LITERAL_RE = re.compile(r"^\{[\d]+\}$")
def _extract_fetch_data(response):
'''Extract data from the response given to an IMAP FETCH command.'''
# Response might be a tuple containing literal data
# At the moment, we only handle one literal per response. This
# may need to be improved if our code ever asks for something
# more complicated (like RFC822.Header and RFC822.Body)
if type(response) == types.TupleType:
literal = response[1]
response = response[0]
else:
literal = None
# the first item will always be the message number
mo = FETCH_RESPONSE_RE.match(response)
data = {}
if mo is None:
print """IMAP server gave strange fetch response. Please
report this as a bug."""
print response
else:
data["message_number"] = mo.group(1)
response = mo.group(2)
# We support the following FETCH items:
# FLAGS
# INTERNALDATE
# RFC822
# UID
# RFC822.HEADER
# All others are ignored.
for r in [FLAGS_RE, INTERNALDATE_RE, RFC822_RE, UID_RE,
RFC822_HEADER_RE]:
mo = r.search(response)
if mo is not None:
if LITERAL_RE.match(mo.group(2)):
data[mo.group(1)] = literal
else:
data[mo.group(1)] = mo.group(2)
return data
class IMAPSession(BaseIMAP):
'''A class extending the IMAP4 class, with a few optimizations'''
def __init__(self, server, port, debug=0, do_expunge=False):
try:
BaseIMAP.__init__(self, server, port)
except:
# A more specific except would be good here, but I get
# (in Python 2.2) a generic 'error' and a 'gaierror'
# if I pass a valid domain that isn't an IMAP server
# or invalid domain (respectively)
print "Invalid server or port, please check these settings."
sys.exit(-1)
self.debug = debug
# For efficiency, we remember which folder we are currently
# in, and only send a select command to the IMAP server if
# we want to *change* folders. This function is used by
# both IMAPMessage and IMAPFolder.
self.current_folder = None
self.do_expunge = do_expunge
def login(self, username, pwd):
try:
BaseIMAP.login(self, username, pwd) # superclass login
except BaseIMAP.error, e:
if str(e) == "permission denied":
print "There was an error logging in to the IMAP server."
print "The userid and/or password may be incorrect."
sys.exit()
else:
raise
def logout(self):
# sign off
if self.do_expunge:
self.expunge()
BaseIMAP.logout(self) # superclass logout
def SelectFolder(self, folder):
'''A method to point ensuing imap operations at a target folder'''
if self.current_folder != folder:
if self.current_folder != None:
if self.do_expunge:
# It is faster to do close() than a single
# expunge when we log out (because expunge returns
# a list of all the deleted messages, that we don't do
# anything with)
imap.close()
# We *always* use SELECT and not EXAMINE, because this
# speeds things up considerably.
response = self.select(folder, None)
if response[0] != "OK":
print "Invalid response to select %s:\n%s" % (folder,
response)
sys.exit(-1)
self.current_folder = folder
return response
def folder_list(self):
'''Return a alphabetical list of all folders available on the
server'''
response = self.list()
if response[0] != "OK":
return []
all_folders = response[1]
folders = []
for fol in all_folders:
# Sigh. Some servers may give us back the folder name as a
# literal, so we need to crunch this out.
if isinstance(fol, ()):
r = re.compile(r"{\d+}")
m = r.search(fol[0])
if not m:
# Something is wrong here! Skip this folder
continue
fol = '%s"%s"' % (fol[0][:m.start()], fol[1])
r = re.compile(r"\(([\w\\ ]*)\) ")
m = r.search(fol)
if not m:
# Something is not good with this folder, so skip it.
continue
name_attributes = fol[:m.end()-1]
# IMAP is a truly odd protocol. The delimiter is
# only the delimiter for this particular folder - each
# folder *may* have a different delimiter
self.folder_delimiter = fol[m.end()+1:m.end()+2]
# a bit of a hack, but we really need to know if this is
# the case
if self.folder_delimiter == ',':
print """WARNING: Your imap server uses commas as the folder
delimiter. This may cause unpredictable errors."""
folders.append(fol[m.end()+5:-1])
folders.sort()
return folders
def FindMessage(self, id):
'''A (potentially very expensive) method to find a message with
a given spambayes id (header), and return a message object (no
substance).'''
# If efficiency becomes a concern, what we could do is store a
# dict of key-to-folder, and look in that folder first. (It might
# have moved independantly of us, so we would still have to search
# if we didn't find it). For the moment, we do an search through
# all folders, alphabetically.
for folder_name in self.folder_list():
fol = IMAPFolder(folder_name)
for msg in fol:
if msg.id == id:
return msg
return None
class IMAPMessage(message.SBHeaderMessage):
def __init__(self):
message.Message.__init__(self)
self.folder = None
self.previous_folder = None
self.rfc822_command = "RFC822.PEEK"
self.got_substance = False
def setFolder(self, folder):
self.folder = folder
def _check(self, response, command):
if response[0] != "OK":
print "Invalid response to %s:\n%s" % (command, response)
sys.exit(-1)
def extractTime(self):
# When we create a new copy of a message, we need to specify
# a timestamp for the message. If the message has a valid date
# header we use that. Otherwise, we use the current time.
message_date = self["Date"]
if message_date is not None:
parsed_date = parsedate(message_date)
if parsed_date is not None:
return Time2Internaldate(time.mktime(parsed_date))
else:
return Time2Internaldate(time.time())
def get_substance(self):
'''Retrieve the RFC822 message from the IMAP server and set as the
substance of this message.'''
if self.got_substance:
return
if not self.uid or not self.id:
print "Cannot get substance of message without an id and an UID"
return
imap.SelectFolder(self.folder.name)
# We really want to use RFC822.PEEK here, as that doesn't effect
# the status of the message. Unfortunately, it appears that not
# all IMAP servers support this, even though it is in RFC1730
# Actually, it's not: we should be using BODY.PEEK
try:
response = imap.uid("FETCH", self.uid, self.rfc822_command)
except IMAP4.error:
self.rfc822_command = "RFC822"
response = imap.uid("FETCH", self.uid, self.rfc822_command)
if response[0] != "OK":
self.rfc822_command = "RFC822"
response = imap.uid("FETCH", self.uid, self.rfc822_command)
self._check(response, "uid fetch")
data = _extract_fetch_data(response[1][0])
# Annoyingly, we can't just pass over the RFC822 message to an
# existing message object (like self) and have it parse it. So
# we go through the hoops of creating a new message, and then
# copying over all its internals.
new_msg = email.Parser.Parser().parsestr(data["RFC822"])
self._headers = new_msg._headers
self._unixfrom = new_msg._unixfrom
self._payload = new_msg._payload
self._charset = new_msg._charset
self.preamble = new_msg.preamble
self.epilogue = new_msg.epilogue
self._default_type = new_msg._default_type
if not self.has_key(options["Headers", "mailid_header_name"]):
self[options["Headers", "mailid_header_name"]] = self.id
self.got_substance = True
if options["globals", "verbose"]:
sys.stdout.write(chr(8) + "*")
def MoveTo(self, dest):
'''Note that message should move to another folder. No move is
carried out until Save() is called, for efficiency.'''
if self.previous_folder is None:
self.previous_folder = self.folder
self.folder = dest
def Save(self):
'''Save message to imap server.'''
# we can't actually update the message with IMAP
# so what we do is create a new message and delete the old one
if self.folder is None:
raise RuntimeError, """Can't save a message that doesn't
have a folder."""
if not self.id:
raise RuntimeError, """Can't save a message that doesn't have
an id."""
response = imap.uid("FETCH", self.uid, "(FLAGS INTERNALDATE)")
self._check(response, 'fetch (flags internaldate)')
data = _extract_fetch_data(response[1][0])
if data.has_key("INTERNALDATE"):
msg_time = data["INTERNALDATE"]
else:
msg_time = self.extractTime()
if data.has_key("FLAGS"):
flags = data["FLAGS"]
# The \Recent flag can be fetched, but cannot be stored
# We must remove it from the list if it is there.
flags = re.sub(r"\\Recent ?|\\ ?Recent", "", flags)
else:
flags = None
response = imap.append(self.folder.name, flags,
msg_time, self.as_string())
if response[0] == "NO":
# This may be because we have tried to set an invalid flag.
# Try again, losing all the flag information, but warn the
# user that this has happened.
response = imap.append(self.folder.name, None, msg_time,
self.as_string())
if response[0] == "OK":
print "WARNING: Could not append flags: %s" % (flags,)
self._check(response, 'append')
if self.previous_folder is None:
imap.SelectFolder(self.folder.name)
else:
imap.SelectFolder(self.previous_folder.name)
self.previous_folder = None
response = imap.uid("STORE", self.uid, "+FLAGS.SILENT", "(\\Deleted)")
self._check(response, 'store')
# We need to update the uid, as it will have changed.
# Although we don't use the UID to keep track of messages, we do
# have to use it for IMAP operations.
imap.SelectFolder(self.folder.name)
response = imap.uid("SEARCH", "(UNDELETED HEADER " + \
options["Headers", "mailid_header_name"] + \
" " + self.id + ")")
self._check(response, 'search')
new_id = response[1][0]
# Let's hope it doesn't, but, just in case, if the search
# turns up empty, we make the assumption that the new
# message is the last one with a recent flag
if new_id == "":
response = imap.uid("SEARCH", "RECENT")
new_id = response[1][0]
if new_id.find(' ') > -1:
ids = new_id.split(' ')
new_id = ids[-1]
# Ok, now we're in trouble if we still haven't found it.
# We make a huge assumption that the new message is the one
# with the highest UID (they are sequential, so this will be
# ok as long as another message hasn't also arrived)
if new_id == "":
response = imap.uid("SEARCH", "ALL")
new_id = response[1][0]
if new_id.find(' ') > -1:
ids = new_id.split(' ')
new_id = ids[-1]
self.uid = new_id
# This performs a similar function to email.message_from_string()
def imapmessage_from_string(s, _class=IMAPMessage, strict=False):
return email.message_from_string(s, _class, strict)
class IMAPFolder(object):
def __init__(self, folder_name):
self.name = folder_name
# Unique names for cached messages - see _generate_id below.
self.lastBaseMessageName = ''
self.uniquifier = 2
def __cmp__(self, obj):
'''Two folders are equal if their names are equal'''
if obj is None:
return False
return cmp(self.name, obj.name)
def _check(self, response, command):
if response[0] != "OK":
print "Invalid response to %s:\n%s" % (command, response)
sys.exit(-1)
def __iter__(self):
'''IMAPFolder is iterable'''
for key in self.keys():
try:
yield self[key]
except KeyError:
pass
def recent_uids(self):
'''Returns uids for all the messages in the folder that
are flagged as recent, but not flagged as deleted.'''
imap.SelectFolder(self.name, True)
response = imap.uid("SEARCH", "RECENT UNDELETED")
self._check(response, "SEARCH RECENT UNDELETED")
return response[1][0].split(' ')
def keys(self):
'''Returns *uids* for all the messages in the folder not
marked as deleted.'''
imap.SelectFolder(self.name)
response = imap.uid("SEARCH", "UNDELETED")
self._check(response, "SEARCH UNDELETED")
if response[1][0] == "":
return []
return response[1][0].split(' ')
def __getitem__(self, key):
'''Return message (no substance) matching the given *uid*.'''
# We don't retrieve the substances of the message here - you need
# to call msg.get_substance() to do that.
imap.SelectFolder(self.name)
# Using RFC822.HEADER.LINES would be better here, but it seems
# that not all servers accept it, even though it is in the RFC
response = imap.uid("FETCH", key, "RFC822.HEADER")
self._check(response, "uid fetch header")
data = _extract_fetch_data(response[1][0])
msg = IMAPMessage()
msg.setFolder(self)
msg.uid = key
r = re.compile(re.escape(options["Headers",
"mailid_header_name"]) + \
"\:\s*(\d+(\-\d)?)")
mo = r.search(data["RFC822.HEADER"])
if mo is None:
msg.setId(self._generate_id())
# Unfortunately, we now have to re-save this message, so that
# our id is stored on the IMAP server. Before anyone suggests
# it, we can't store it as a flag, because user-defined flags
# aren't supported by all IMAP servers.
# This will need to be done once per message.
msg.get_substance()
msg.Save()
else:
msg.setId(mo.group(1))
if options["globals", "verbose"]:
sys.stdout.write(".")
return msg
# Lifted straight from pop3proxy.py (under the name getNewMessageName)
def _generate_id(self):
# The message id is the time it arrived, with a uniquifier
# appended if two arrive within one clock tick of each other.
messageName = "%10.10d" % long(time.time())
if messageName == self.lastBaseMessageName:
messageName = "%s-%d" % (messageName, self.uniquifier)
self.uniquifier += 1
else:
self.lastBaseMessageName = messageName
self.uniquifier = 2
return messageName
def Train(self, classifier, isSpam):
'''Train folder as spam/ham'''
num_trained = 0
for msg in self:
if msg.GetTrained() == (not isSpam):
msg.get_substance()
msg.delSBHeaders()
classifier.unlearn(msg.asTokens(), not isSpam)
# Once the message has been untrained, it's training memory
# should reflect that on the off chance that for some reason
# the training breaks, which happens on occasion (the
# tokenizer is not yet perfect)
msg.RememberTrained(None)
if msg.GetTrained() is None:
msg.get_substance()
msg.delSBHeaders()
classifier.learn(msg.asTokens(), isSpam)
num_trained += 1
msg.RememberTrained(isSpam)
if isSpam:
move_opt_name = "move_trained_spam_to_folder"
else:
move_opt_name = "move_trained_ham_to_folder"
if options["imap", move_opt_name] != "":
msg.MoveTo(IMAPFolder(options["imap",
move_opt_name]))
msg.Save()
return num_trained
def Filter(self, classifier, spamfolder, unsurefolder):
count = {}
count["ham"] = 0
count["spam"] = 0
count["unsure"] = 0
for msg in self:
if msg.GetClassification() is None:
msg.get_substance()
(prob, clues) = classifier.spamprob(msg.asTokens(),
evidence=True)
# add headers and remember classification
msg.addSBHeaders(prob, clues)
cls = msg.GetClassification()
if cls == options["Hammie", "header_ham_string"]:
# we leave ham alone
count["ham"] += 1
elif cls == options["Hammie", "header_spam_string"]:
msg.MoveTo(spamfolder)
count["spam"] += 1
else:
msg.MoveTo(unsurefolder)
count["unsure"] += 1
msg.Save()
return count
class IMAPFilter(object):
def __init__(self, classifier):
self.spam_folder = IMAPFolder(options["imap", "spam_folder"])
self.unsure_folder = IMAPFolder(options["imap", "unsure_folder"])
self.classifier = classifier
def Train(self):
if options["globals", "verbose"]:
t = time.time()
total_ham_trained = 0
total_spam_trained = 0
if options["imap", "ham_train_folders"] != "":
ham_training_folders = options["imap", "ham_train_folders"]
for fol in ham_training_folders:
# Select the folder to make sure it exists
imap.SelectFolder(fol)
if options['globals', 'verbose']:
print " Training ham folder %s" % (fol)
folder = IMAPFolder(fol)
num_ham_trained = folder.Train(self.classifier, False)
total_ham_trained += num_ham_trained
if options['globals', 'verbose']:
print " %s trained." % (num_ham_trained)
if options["imap", "spam_train_folders"] != "":
spam_training_folders = options["imap", "spam_train_folders"]
for fol in spam_training_folders:
# Select the folder to make sure it exists
imap.SelectFolder(fol)
if options['globals', 'verbose']:
print " Training spam folder %s" % (fol)
folder = IMAPFolder(fol)
num_spam_trained = folder.Train(self.classifier, True)
total_spam_trained += num_spam_trained
if options['globals', 'verbose']:
print " %s trained." % (num_spam_trained)
if total_ham_trained or total_spam_trained:
self.classifier.store()
if options["globals", "verbose"]:
print "Training took %s seconds, %s messages were trained" \
% (time.time() - t, total_ham_trained + total_spam_trained)
def Filter(self):
if options["globals", "verbose"]:
t = time.time()
count = None
# Select the spam folder and unsure folder to make sure they exist
imap.SelectFolder(self.spam_folder.name)
imap.SelectFolder(self.unsure_folder.name)
for filter_folder in options["imap", "filter_folders"]:
# Select the folder to make sure it exists
imap.SelectFolder(filter_folder)
folder = IMAPFolder(filter_folder)
count = folder.Filter(self.classifier, self.spam_folder,
self.unsure_folder)
if options["globals", "verbose"]:
if count is not None:
print "\nClassified %s ham, %s spam, and %s unsure." % \
(count["ham"], count["spam"], count["unsure"])
print "Classifying took", time.time() - t, "seconds."
def run():
global imap
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbtcvpl:e:i:d:D:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = options["Storage", "persistent_storage_file"]
useDBM = options["Storage", "persistent_use_database"]
doTrain = False
doClassify = False
doExpunge = options["imap", "expunge"]
imapDebug = 0
sleepTime = 0
promptForPass = False
launchUI = False
server = ""
username = ""
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == "-b":
launchUI = True
elif opt == '-t':
doTrain = True
elif opt == '-p':
promptForPass = True
elif opt == '-c':
doClassify = True
elif opt == '-v':
options["globals", "verbose"] = True
elif opt == '-e':
if arg == 'y':
doExpunge = True
else:
doExpunge = False
elif opt == '-i':
imapDebug = int(arg)
elif opt == '-l':
sleepTime = int(arg) * 60
# Let the user know what they are using...
print get_version_string("IMAP Filter")
print "and engine %s.\n" % (get_version_string(),)
if not (doClassify or doTrain or launchUI):
print "-b, -c, or -t operands must be specified."
print "Please use the -h operand for help."
sys.exit()
if (launchUI and (doClassify or doTrain)):
print """
-b option is exclusive with -c and -t options.
The user interface will be launched, but no classification
or training will be performed."""
bdbname = os.path.expanduser(bdbname)
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
classifier = storage.open_storage(bdbname, useDBM)
if options["globals", "verbose"]:
print "Done."
if options["imap", "server"]:
# The options class is ahead of us here:
# it knows that imap:server will eventually be able to have
# multiple values, but for the moment, we just use the first one
server = options["imap", "server"]
if len(server) > 0:
server = server[0]
username = options["imap", "username"]
if len(username) > 0:
username = username[0]
if not promptForPass:
pwd = options["imap", "password"]
if len(pwd) > 0:
pwd = pwd[0]
else:
pwd = None
if not launchUI:
print "You need to specify both a server and a username."
sys.exit()
if promptForPass:
pwd = getpass()
if server.find(':') > -1:
server, port = server.split(':', 1)
port = int(port)
else:
if options["imap", "use_ssl"]:
port = 993
else:
port = 143
imap_filter = IMAPFilter(classifier)
# Web interface
if launchUI:
if server != "":
imap = IMAPSession(server, port, imapDebug, doExpunge)
httpServer = UserInterfaceServer(options["html_ui", "port"])
httpServer.register(IMAPUserInterface(classifier, imap, pwd))
Dibbler.run(launchBrowser=launchUI)
else:
while True:
imap = IMAPSession(server, port, imapDebug, doExpunge)
imap.login(username, pwd)
if doTrain:
if options["globals", "verbose"]:
print "Training"
imap_filter.Train()
if doClassify:
if options["globals", "verbose"]:
print "Classifying"
imap_filter.Filter()
imap.logout()
if sleepTime:
time.sleep(sleepTime)
else:
break
if __name__ == '__main__':
run()
--- NEW FILE: sb_mailsort.py ---
#! /usr/bin/env python
"""\
To train:
%(program)s -t ham.mbox spam.mbox
To filter mail (using .forward or .qmail):
|%(program)s Maildir/ Mail/Spam/
To print the score and top evidence for a message or messages:
%(program)s -s message [message ...]
"""
SPAM_CUTOFF = 0.57
SIZE_LIMIT = 5000000 # messages larger are not analyzed
BLOCK_SIZE = 10000
RC_DIR = "~/.spambayes"
DB_FILE = RC_DIR + "/wordprobs.cdb"
CONFIG_FILE = RC_DIR + "/bayescustomize.ini"
import sys
import os
import getopt
import email
import time
import signal
import socket
import email
DB_FILE = os.path.expanduser(DB_FILE)
def import_spambayes():
global mboxutils, CdbClassifier, tokenize
if not os.environ.has_key('BAYESCUSTOMIZE'):
os.environ['BAYESCUSTOMIZE'] = os.path.expanduser(CONFIG_FILE)
from spambayes import mboxutils
from spambayes.cdb_classifier import CdbClassifier
from spambayes.tokenizer import tokenize
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
program = sys.argv[0] # For usage(); referenced by docstring above
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def maketmp(dir):
hostname = socket.gethostname()
pid = os.getpid()
fd = -1
for x in xrange(200):
filename = "%d.%d.%s" % (time.time(), pid, hostname)
pathname = "%s/tmp/%s" % (dir, filename)
try:
fd = os.open(pathname, os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0600)
except IOError, exc:
if exc[i] not in (errno.EINT, errno.EEXIST):
raise
else:
break
time.sleep(2)
if fd == -1:
raise SystemExit, "could not create a mail file"
return (os.fdopen(fd, "wb"), pathname, filename)
def train(bayes, msgs, is_spam):
"""Train bayes with all messages from a mailbox."""
mbox = mboxutils.getmbox(msgs)
for msg in mbox:
bayes.learn(tokenize(msg), is_spam)
def train_messages(ham_name, spam_name):
"""Create database using messages."""
rc_dir = os.path.expanduser(RC_DIR)
if not os.path.exists(rc_dir):
print "Creating", RC_DIR, "directory..."
os.mkdir(rc_dir)
bayes = CdbClassifier()
print 'Training with ham...'
train(bayes, ham_name, False)
print 'Training with spam...'
train(bayes, spam_name, True)
print 'Update probabilities and writing DB...'
db = open(DB_FILE, "wb")
bayes.save_wordinfo(db)
db.close()
print 'done'
def filter_message(hamdir, spamdir):
signal.signal(signal.SIGALRM, lambda s: sys.exit(1))
signal.alarm(24 * 60 * 60)
# write message to temporary file (must be on same partition)
tmpfile, pathname, filename = maketmp(hamdir)
try:
tmpfile.write(os.environ.get("DTLINE", "")) # delivered-to line
bytes = 0
blocks = []
while 1:
block = sys.stdin.read(BLOCK_SIZE)
if not block:
break
bytes += len(block)
if bytes < SIZE_LIMIT:
blocks.append(block)
tmpfile.write(block)
tmpfile.close()
if bytes < SIZE_LIMIT:
msgdata = ''.join(blocks)
del blocks
msg = email.message_from_string(msgdata)
del msgdata
bayes = CdbClassifier(open(DB_FILE, 'rb'))
prob = bayes.spamprob(tokenize(msg))
else:
prob = 0.0
if prob > SPAM_CUTOFF:
os.rename(pathname, "%s/new/%s" % (spamdir, filename))
else:
os.rename(pathname, "%s/new/%s" % (hamdir, filename))
except:
os.unlink(pathname)
raise
def print_message_score(msg_name, msg_fp):
msg = email.message_from_file(msg_fp)
bayes = CdbClassifier(open(DB_FILE, 'rb'))
prob, evidence = bayes.spamprob(tokenize(msg), evidence=True)
print msg_name, prob
for word, prob in evidence:
print ' ', `word`, prob
def main():
global DB_FILE, CONFIG_FILE
try:
opts, args = getopt.getopt(sys.argv[1:], 'tsd:c:')
except getopt.error, msg:
usage(2, msg)
mode = 'sort'
for opt, val in opts:
if opt == '-t':
mode = 'train'
elif opt == '-s':
mode = 'score'
elif opt == '-d':
DB_FILE = val
elif opt == '-c':
CONFIG_FILE = val
else:
assert 0, 'invalid option'
import_spambayes()
if mode == 'sort':
if len(args) != 2:
usage(2, 'wrong number of arguments')
filter_message(args[0], args[1])
elif mode == 'train':
if len(args) != 2:
usage(2, 'wrong number of arguments')
train_messages(args[0], args[1])
elif mode == 'score':
if args:
for msg in args:
print_message_score(msg, open(msg))
else:
print_message_score('', sys.stdin)
if __name__ == "__main__":
main()
--- NEW FILE: sb_mboxtrain.py ---
#! /usr/bin/env python
### Train spambayes on all previously-untrained messages in a mailbox.
###
### This keeps track of messages it's already trained by adding an
### X-Spambayes-Trained: header to each one. Then, if you move one to
### another folder, it will retrain that message. You would want to run
### this from a cron job on your server.
"""Usage: %(program)s [OPTIONS] ...
Where OPTIONS is one or more of:
-h
show usage and exit
-d DBNAME
use the DBM store. A DBM file is larger than the pickle and
creating it is slower, but loading it is much faster,
especially for large word databases. Recommended for use with
hammiefilter or any procmail-based filter.
-D DBNAME
use the pickle store. A pickle is smaller and faster to create,
but much slower to load. Recommended for use with pop3proxy and
hammiesrv.
-g PATH
mbox or directory of known good messages (non-spam) to train on.
Can be specified more than once.
-s PATH
mbox or directory of known spam messages to train on.
Can be specified more than once.
-f
force training, ignoring the trained header. Use this if you
need to rebuild your database from scratch.
-q
quiet mode; no output
-n train mail residing in "new" directory, in addition to "cur" directory,
which is always trained (Maildir only)
-r remove mail which was trained on (Maildir only)
"""
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import sys, os, getopt
from spambayes import hammie, mboxutils
from spambayes.Options import options
program = sys.argv[0]
TRAINED_HDR = "X-Spambayes-Trained"
loud = True
def msg_train(h, msg, is_spam, force):
"""Train bayes with a single message."""
# XXX: big hack -- why is email.Message unable to represent
# multipart/alternative?
try:
msg.as_string()
except TypeError:
# We'll be unable to represent this as text :(
return False
if is_spam:
spamtxt = options["Headers", "header_spam_string"]
else:
spamtxt = options["Headers", "header_ham_string"]
oldtxt = msg.get(TRAINED_HDR)
if force:
# Train no matter what.
if oldtxt != None:
del msg[TRAINED_HDR]
elif oldtxt == spamtxt:
# Skip this one, we've already trained with it.
return False
elif oldtxt != None:
# It's been trained, but as something else. Untrain.
del msg[TRAINED_HDR]
h.untrain(msg, not is_spam)
h.train(msg, is_spam)
msg.add_header(TRAINED_HDR, spamtxt)
return True
def maildir_train(h, path, is_spam, force, removetrained):
"""Train bayes with all messages from a maildir."""
if loud: print " Reading %s as Maildir" % (path,)
import time
import socket
pid = os.getpid()
host = socket.gethostname()
counter = 0
trained = 0
for fn in os.listdir(path):
cfn = os.path.join(path, fn)
tfn = os.path.normpath(os.path.join(path, "..", "tmp",
"%d.%d_%d.%s" % (time.time(), pid,
counter, host)))
if (os.path.isdir(cfn)):
continue
counter += 1
if loud:
sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(cfn, "rb")
msg = mboxutils.get_message(f)
f.close()
if not msg_train(h, msg, is_spam, force):
continue
trained += 1
f = file(tfn, "wb")
f.write(msg.as_string())
f.close()
# XXX: This will raise an exception on Windows. Do any Windows
# people actually use Maildirs?
os.rename(tfn, cfn)
if (removetrained):
os.unlink(cfn)
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def mbox_train(h, path, is_spam, force):
"""Train bayes with a Unix mbox"""
if loud: print " Reading as Unix mbox"
import mailbox
import fcntl
import tempfile
# Open and lock the mailbox. Some systems require it be opened for
# writes in order to assert an exclusive lock.
f = file(path, "r+b")
fcntl.flock(f, fcntl.LOCK_EX)
mbox = mailbox.PortableUnixMailbox(f, mboxutils.get_message)
outf = os.tmpfile()
counter = 0
trained = 0
for msg in mbox:
counter += 1
if loud:
sys.stdout.write(" %s\r" % counter)
sys.stdout.flush()
if msg_train(h, msg, is_spam, force):
trained += 1
# Write it out with the Unix "From " line
outf.write(msg.as_string(True))
outf.seek(0)
try:
os.ftruncate(f.fileno(), 0)
f.seek(0)
except:
# If anything goes wrong, don't try to write
print "Problem truncating mbox--nothing written"
raise
try:
for line in outf.xreadlines():
f.write(line)
except:
print >> sys.stderr ("Problem writing mbox! Sorry, "
"I tried my best, but your mail "
"may be corrupted.")
raise
fcntl.lockf(f, fcntl.LOCK_UN)
f.close()
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def mhdir_train(h, path, is_spam, force):
"""Train bayes with an mh directory"""
if loud: print " Reading as MH mailbox"
import glob
counter = 0
trained = 0
for fn in glob.glob(os.path.join(path, "[0-9]*")):
counter += 1
cfn = fn
tfn = os.path.join(path, "spambayes.tmp")
if loud:
sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(fn, "rb")
msg = mboxutils.get_message(f)
f.close()
msg_train(h, msg, is_spam, force)
trained += 1
f = file(tfn, "wb")
f.write(msg.as_string())
f.close()
# XXX: This will raise an exception on Windows. Do any Windows
# people actually use MH directories?
os.rename(tfn, cfn)
if loud:
print (" Trained %d out of %d messages " %
(trained, counter))
def train(h, path, is_spam, force, trainnew, removetrained):
if not os.path.exists(path):
raise ValueError("Nonexistent path: %s" % path)
elif os.path.isfile(path):
mbox_train(h, path, is_spam, force)
elif os.path.isdir(os.path.join(path, "cur")):
maildir_train(h, os.path.join(path, "cur"), is_spam, force, removetrained)
if trainnew:
maildir_train(h, os.path.join(path, "new"), is_spam, force, removetrained)
elif os.path.isdir(path):
mhdir_train(h, path, is_spam, force)
else:
raise ValueError("Unable to determine mailbox type: " + path)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def main():
"""Main program; parse options and go."""
global loud
try:
opts, args = getopt.getopt(sys.argv[1:], 'hfqnrd:D:g:s:')
except getopt.error, msg:
usage(2, msg)
if not opts:
usage(2, "No options given")
pck = None
usedb = None
force = False
trainnew = False
removetrained = False
good = []
spam = []
for opt, arg in opts:
if opt == '-h':
usage(0)
elif opt == "-f":
force = True
elif opt == "-n":
trainnew = True
elif opt == "-q":
loud = False
elif opt == '-g':
good.append(arg)
elif opt == '-s':
spam.append(arg)
elif opt == "-r":
removetrained = True
elif opt == "-d":
usedb = True
pck = arg
elif opt == "-D":
usedb = False
pck = arg
if args:
usage(2, "Positional arguments not allowed")
if usedb == None:
usage(2, "Must specify one of -d or -D")
h = hammie.open(pck, usedb, "c")
for g in good:
if loud: print "Training ham (%s):" % g
train(h, g, False, force, trainnew, removetrained)
save = True
for s in spam:
if loud: print "Training spam (%s):" % s
train(h, s, True, force, trainnew, removetrained)
save = True
if save:
h.store()
if __name__ == "__main__":
main()
--- NEW FILE: sb_notesfilter.py ---
#! /usr/bin/env python
'''notesfilter.py - Lotus Notes Spambayes interface.
Classes:
Abstract:
This module uses Spambayes as a filter against a Lotus Notes mail
database. The Notes client must be running when this process is
executed.
It requires a Notes folder, named as a parameter, with four
subfolders:
Spam
Ham
Train as Spam
Train as Ham
Depending on the execution parameters, it will do any or all of the
following steps, in the order given.
1. Train Spam from the Train as Spam folder (-t option)
2. Train Ham from the Train as Ham folder (-t option)
3. Replicate (-r option)
4. Classify the inbox (-c option)
Mail that is to be trained as spam should be manually moved to
that folder by the user. Likewise mail that is to be trained as
ham. After training, spam is moved to the Spam folder and ham is
moved to the Ham folder.
Replication takes place if a remote server has been specified.
This step may take a long time, depending on replication
parameters and how much information there is to download, as well
as line speed and server load. Please be patient if you run with
replication. There is currently no progress bar or anything like
that to tell you that it's working, but it is and will complete
eventually. There is also no mechanism for notifying you that the
replication failed. If it did, there is no harm done, and the program
will continue execution.
Mail that is classified as Spam is moved from the inbox to the
Train as Spam folder. You should occasionally review your Spam
folder for Ham that has mistakenly been classified as Spam. If
there is any there, move it to the Train as Ham folder, so
Spambayes will be less likely to make this mistake again.
Mail that is classified as Ham or Unsure is left in the inbox.
There is currently no means of telling if a mail was classified as
Ham or Unsure.
You should occasionally select some Ham and move it to the Train
as Ham folder, so Spambayes can tell the difference between Spam
and Ham. The goal is to maintain a relative balance between the
number of Spam and the number of Ham that have been trained into
the database. These numbers are reported every time this program
executes. However, if the amount of Spam you receive far exceeds
the amount of Ham you receive, it may be very difficult to
maintain this balance. This is not a matter of great concern.
Spambayes will still make very few mistakes in this circumstance.
But, if this is the case, you should review your Spam folder for
falsely classified Ham, and retrain those that you find, on a
regular basis. This will prevent statistical error accumulation,
which if allowed to continue, would cause Spambayes to tend to
classify everything as Spam.
Because there is no programmatic way to determine if a particular
mail has been previously processed by this classification program,
it keeps a pickled dictionary of notes mail ids, so that once a
mail has been classified, it will not be classified again. The
non-existence of is index file, named .sbindex,
indicates to the system that this is an initialization execution.
Rather than classify the inbox in this case, the contents of the
inbox are placed in the index to note the 'starting point' of the
system. After that, any new messages in the inbox are eligible
for classification.
Usage:
notesfilter [options]
note: option values with spaces in them must be enclosed
in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-l dbname : database filename of local mail replica
e.g. localmail.nsf
-r server : server address of the server mail database
e.g. d27ml602/27/M/IBM
if specified, will initiate a replication
-f folder : Name of spambayes folder
must have subfolders: Spam
Ham
Train as Spam
Train as Ham
-t : train contents of Train as Spam and Train as Ham
-c : classify inbox
-h : help
-p : prompt "Press Enter to end" before ending
This is useful for automated executions where the
statistics output would otherwise be lost when the
window closes.
Examples:
Replicate and classify inbox
notesfilter -c -d notesbayes -r mynoteserv -l mail.nsf -f Spambayes
Train Spam and Ham, then classify inbox
notesfilter -t -c -d notesbayes -l mail.nsf -f Spambayes
Replicate, then classify inbox
notesfilter -c -d test7 -l mail.nsf -r nynoteserv -f Spambayes
To Do:
o Dump/purge notesindex file
o Create correct folders if they do not exist
o Options for some of this stuff?
o pop3proxy style training/configuration interface?
o parameter to retrain?
o Suggestions?
'''
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tim Stone "
__credits__ = "Mark Hammond, for his remarkable win32 modules."
from __future__ import generators
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
def bool(val):
return not not val
import sys
from spambayes import tokenizer, storage
from spambayes.Options import options
import cPickle as pickle
import errno
import win32com.client
import pywintypes
import getopt
def classifyInbox(v, vmoveto, bayes, ldbname, notesindex):
# the notesindex hash ensures that a message is looked at only once
if len(notesindex.keys()) == 0:
firsttime = 1
else:
firsttime = 0
docstomove = []
numham = 0
numspam = 0
numuns = 0
numdocs = 0
doc = v.GetFirstDocument()
while doc:
nid = doc.NOTEID
if firsttime:
notesindex[nid] = 'never classified'
else:
if not notesindex.has_key(nid):
numdocs += 1
# Notes returns strings in unicode, and the Python
# uni-decoder has trouble with these strings when
# you try to print them. So don't...
# The com interface returns basic data types as tuples
# only, thus the subscript on GetItemValue
try:
subj = doc.GetItemValue('Subject')[0]
except:
subj = 'No Subject'
try:
body = doc.GetItemValue('Body')[0]
except:
body = 'No Body'
message = "Subject: %s\r\n\r\n%s" % (subj, body)
# generate_long_skips = True blows up on occasion,
# probably due to this unicode problem.
options["Tokenizer", "generate_long_skips"] = False
tokens = tokenizer.tokenize(message)
prob, clues = bayes.spamprob(tokens, evidence=True)
if prob < options["Categorization", "ham_cutoff"]:
disposition = options["Hammie", "header_ham_string"]
numham += 1
elif prob > options["Categorization", "spam_cutoff"]:
disposition = options["Hammie", "header_spam_string"]
docstomove += [doc]
numspam += 1
else:
disposition = options["Hammie", "header_unsure_string"]
numuns += 1
notesindex[nid] = 'classified'
try:
print "%s spamprob is %s" % (subj[:30], prob)
except UnicodeError:
print " spamprob is %s" % (prob)
doc = v.GetNextDocument(doc)
# docstomove list is built because moving documents in the middle of
# the classification loop looses the iterator position
for doc in docstomove:
doc.RemoveFromFolder(v.Name)
doc.PutInFolder(vmoveto.Name)
print "%s documents processed" % (numdocs)
print " %s classified as spam" % (numspam)
print " %s classified as ham" % (numham)
print " %s classified as unsure" % (numuns)
def processAndTrain(v, vmoveto, bayes, is_spam, notesindex):
if is_spam:
str = options["Hammie", "header_spam_string"]
else:
str = options["Hammie", "header_ham_string"]
print "Training %s" % (str)
docstomove = []
doc = v.GetFirstDocument()
while doc:
try:
subj = doc.GetItemValue('Subject')[0]
except:
subj = 'No Subject'
try:
body = doc.GetItemValue('Body')[0]
except:
body = 'No Body'
message = "Subject: %s\r\n%s" % (subj, body)
options["Tokenizer", "generate_long_skips"] = False
tokens = tokenizer.tokenize(message)
nid = doc.NOTEID
if notesindex.has_key(nid):
trainedas = notesindex[nid]
if trainedas == options["Hammie", "header_spam_string"] and \
not is_spam:
# msg is trained as spam, is to be retrained as ham
bayes.unlearn(tokens, True)
elif trainedas == options["Hammie", "header_ham_string"] and \
is_spam:
# msg is trained as ham, is to be retrained as spam
bayes.unlearn(tokens, False)
bayes.learn(tokens, is_spam)
notesindex[nid] = str
docstomove += [doc]
doc = v.GetNextDocument(doc)
for doc in docstomove:
doc.RemoveFromFolder(v.Name)
doc.PutInFolder(vmoveto.Name)
print "%s documents trained" % (len(docstomove))
def run(bdbname, useDBM, ldbname, rdbname, foldname, doTrain, doClassify):
if useDBM:
bayes = storage.DBDictClassifier(bdbname)
else:
bayes = storage.PickledClassifier(bdbname)
try:
fp = open("%s.sbindex" % (ldbname), 'rb')
except IOError, e:
if e.errno != errno.ENOENT: raise
notesindex = {}
print "%s.sbindex file not found, this is a first time run" \
% (ldbname)
print "No classification will be performed"
else:
notesindex = pickle.load(fp)
fp.close()
sess = win32com.client.Dispatch("Lotus.NotesSession")
try:
sess.initialize()
except pywintypes.com_error:
print "Session aborted"
sys.exit()
db = sess.GetDatabase("",ldbname)
vinbox = db.getView('($Inbox)')
vspam = db.getView("%s\Spam" % (foldname))
vham = db.getView("%s\Ham" % (foldname))
vtrainspam = db.getView("%s\Train as Spam" % (foldname))
vtrainham = db.getView("%s\Train as Ham" % (foldname))
if doTrain:
processAndTrain(vtrainspam, vspam, bayes, True, notesindex)
# for some reason, using inbox as a target here loses the mail
processAndTrain(vtrainham, vham, bayes, False, notesindex)
if rdbname:
print "Replicating..."
db.Replicate(rdbname)
print "Done"
if doClassify:
classifyInbox(vinbox, vtrainspam, bayes, ldbname, notesindex)
print "The Spambayes database currently has %s Spam and %s Ham" \
% (bayes.nspam, bayes.nham)
bayes.store()
fp = open("%s.sbindex" % (ldbname), 'wb')
pickle.dump(notesindex, fp)
fp.close()
if __name__ == '__main__':
try:
opts, args = getopt.getopt(sys.argv[1:], 'htcpd:D:l:r:f:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = None # bayes database name
ldbname = None # local notes database name
rdbname = None # remote notes database location
sbfname = None # spambayes folder name
doTrain = False
doClassify = False
doPrompt = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == '-l':
ldbname = arg
elif opt == '-r':
rdbname = arg
elif opt == '-f':
sbfname = arg
elif opt == '-t':
doTrain = True
elif opt == '-c':
doClassify = True
elif opt == '-p':
doPrompt = True
if (bdbname and ldbname and sbfname and (doTrain or doClassify)):
run(bdbname, useDBM, ldbname, rdbname, \
sbfname, doTrain, doClassify)
if doPrompt:
try:
key = input("Press Enter to end")
except SyntaxError:
pass
else:
print >>sys.stderr, __doc__
--- NEW FILE: sb_pop3dnd.py ---
#!/usr/bin/env python
from __future__ import generators
"""
Overkill (someone *please* come up with something to call this script!)
This application is a twisted cross between a POP3 proxy and an IMAP
server. It sits between your mail client and your POP3 server (like any
other POP3 proxy). While messages classified as ham are simply passed
through the proxy, messages that are classified as spam or unsure are
intercepted and passed to the IMAP server. The IMAP server offers three
folders - one where messages classified as spam end up, one for messages
it is unsure about, and one for training ham.
In other words, to use this application, setup your mail client to connect
to localhost, rather than directly to your POP3 server. Additionally, add
a new IMAP account, also connecting to localhost. Setup the application
via the web interface, and you are ready to go. Good messages will appear
as per normal, but you will also have two new incoming folders, one for
spam and one for ham.
To train SpamBayes, use the spam folder, and the 'train_as_ham' folder.
Any messages in these folders will be trained appropriately. This means
that all messages that SpamBayes classifies as spam will also be trained
as such. If you receive any 'false positives' (ham classified as spam),
you *must* copy the message into the 'train_as_ham' folder to correct the
training. You may also place any saved spam messages you have into this
folder.
So that SpamBayes knows about ham as well as spam, you will also need to
move or copy mail into the 'train_as_ham' folder. These may come from
the unsure folder, or from any other mail you have saved. It is a good
idea to leave messages in the 'train_as_ham' and 'spam' folders, so that
you can retrain from scratch if required. (However, you should always
clear out your unsure folder, preferably moving or copying the messages
into the appropriate training folder).
This SpamBayes application is designed to work with Outlook Express, and
provide the same sort of ease of use as the Outlook plugin. Although the
majority of development and testing has been done with Outlook Express,
any mail client that supports both IMAP and POP3 should be able to use this
application - if the client enables the user to work with an IMAP account
and POP3 account side-by-side (and move messages between them), then it
should work equally as well as Outlook Express.
This module includes the following classes:
o IMAPFileMessage
o IMAPFileMessageFactory
o IMAPMailbox
o SpambayesMailbox
o Trainer
o SpambayesAccount
o SpambayesIMAPServer
o OneParameterFactory
o MyBayesProxy
o MyBayesProxyListener
o IMAPState
"""
todo = """
o Message flags are currently not persisted, but should be. The
IMAPFileMessage class should be extended to do this. The same
goes for the 'internaldate' of the message.
o The RECENT flag should be unset at some point, but when? The
RFC says that a message is recent if this is the first session
to be notified about the message. Perhaps this can be done
simply by *not* persisting this flag - i.e. the flag is always
loaded as not recent, and only new messages are recent. The
RFC says that if it is not possible to determine, then all
messages should be recent, and this is what we currently do.
o The Mailbox should be calling the appropriate listener
functions (currently only newMessages is called on addMessage).
flagsChanged should also be called on store, addMessage, or ???
o We cannot currently get part of a message via the BODY calls
(with the <> operands), or get a part of a MIME message (by
prepending a number). This should be added!
o If the user clicks the 'save and shutdown' button on the web
interface, this will only kill the POP3 proxy and web interface
threads, and not the IMAP server. We need to monitor the thread
that we kick off, and if it dies, we should die too. Need to figure
out how to do this in twisted.
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "All the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
import os
import re
import sys
import md5
import time
import errno
import types
import thread
import getopt
import imaplib
import operator
import StringIO
import email.Utils
from twisted import cred
from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.app import Application
from twisted.internet.defer import maybeDeferred
from twisted.internet.protocol import ServerFactory
from twisted.protocols.imap4 import parseNestedParens, parseIdList
from twisted.protocols.imap4 import IllegalClientResponse, IAccount
from twisted.protocols.imap4 import collapseNestedLists, MessageSet
from twisted.protocols.imap4 import IMAP4Server, MemoryAccount, IMailbox
from twisted.protocols.imap4 import IMailboxListener, collapseNestedLists
# Provide for those that don't have spambayes on their PYTHONPATH
sys.path.insert(-1, os.path.dirname(os.getcwd()))
from spambayes.Options import options
from spambayes.message import Message
from spambayes.tokenizer import tokenize
from spambayes import FileCorpus, Dibbler
from spambayes.Version import get_version_string
from spambayes.ServerUI import ServerUserInterface
from spambayes.UserInterface import UserInterfaceServer
from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState
def ensureDir(dirname):
"""Ensure that the given directory exists - in other words, if it
does not exist, attempt to create it."""
try:
os.mkdir(dirname)
if options["globals", "verbose"]:
print "Creating directory", dirname
except OSError, e:
if e.errno != errno.EEXIST:
raise
class IMAPFileMessage(FileCorpus.FileMessage):
'''IMAP Message that persists as a file system artifact.'''
def __init__(self, file_name, directory):
"""Constructor(message file name, corpus directory name)."""
FileCorpus.FileMessage.__init__(self, file_name, directory)
self.id = file_name
self.directory = directory
self.date = imaplib.Time2Internaldate(time.time())[1:-1]
self.clear_flags()
# IMessage implementation
def getHeaders(self, negate, names):
"""Retrieve a group of message headers."""
headers = {}
if not isinstance(names, tuple):
names = (names,)
for header, value in self.items():
if (header.upper() in names and not negate) or names == ():
headers[header.upper()] = value
return headers
def getFlags(self):
"""Retrieve the flags associated with this message."""
return self._flags_iter()
def _flags_iter(self):
if self.deleted:
yield "\\DELETED"
if self.answered:
yield "\\ANSWERED"
if self.flagged:
yield "\\FLAGGED"
if self.seen:
yield "\\SEEN"
if self.draft:
yield "\\DRAFT"
if self.draft:
yield "\\RECENT"
def getInternalDate(self):
"""Retrieve the date internally associated with this message."""
return self.date
def getBodyFile(self):
"""Retrieve a file object containing the body of this message."""
# Note only body, not headers!
s = StringIO.StringIO()
s.write(self.body())
s.seek(0)
return s
#return file(os.path.join(self.directory, self.id), "r")
def getSize(self):
"""Retrieve the total size, in octets, of this message."""
return len(self.as_string())
def getUID(self):
"""Retrieve the unique identifier associated with this message."""
return self.id
def getSubPart(self, part):
"""Retrieve a MIME sub-message
@type part: C{int}
@param part: The number of the part to retrieve, indexed from 0.
@rtype: Any object implementing C{IMessage}.
@return: The specified sub-part.
"""
# IMessage implementation ends
def clear_flags(self):
"""Set all message flags to false."""
self.deleted = False
self.answered = False
self.flagged = False
self.seen = False
self.draft = False
self.recent = False
def set_flag(self, flag, value):
# invalid flags are ignored
flag = flag.upper()
if flag == "\\DELETED":
self.deleted = value
elif flag == "\\ANSWERED":
self.answered = value
elif flag == "\\FLAGGED":
self.flagged = value
elif flag == "\\SEEN":
self.seen = value
elif flag == "\\DRAFT":
self.draft = value
else:
print "Tried to set invalid flag", flag, "to", value
def flags(self):
"""Return the message flags."""
all_flags = []
if self.deleted:
all_flags.append("\\DELETED")
if self.answered:
all_flags.append("\\ANSWERED")
if self.flagged:
all_flags.append("\\FLAGGED")
if self.seen:
all_flags.append("\\SEEN")
if self.draft:
all_flags.append("\\DRAFT")
if self.draft:
all_flags.append("\\RECENT")
return all_flags
def train(self, classifier, isSpam):
if self.GetTrained() == (not isSpam):
classifier.unlearn(self.asTokens(), not isSpam)
self.RememberTrained(None)
if self.GetTrained() is None:
classifier.learn(self.asTokens(), isSpam)
self.RememberTrained(isSpam)
classifier.store()
def structure(self, ext=False):
"""Body structure data describes the MIME-IMB
format of a message and consists of a sequence of mime type, mime
subtype, parameters, content id, description, encoding, and size.
The fields following the size field are variable: if the mime
type/subtype is message/rfc822, the contained message's envelope
information, body structure data, and number of lines of text; if
the mime type is text, the number of lines of text. Extension fields
may also be included; if present, they are: the MD5 hash of the body,
body disposition, body language."""
s = []
for part in self.walk():
if part.get_content_charset() is not None:
charset = ("charset", part.get_content_charset())
else:
charset = None
part_s = [part.get_main_type(), part.get_subtype(),
charset,
part.get('Content-Id'),
part.get('Content-Description'),
part.get('Content-Transfer-Encoding'),
str(len(part.as_string()))]
#if part.get_type() == "message/rfc822":
# part_s.extend([envelope, body_structure_data,
# part.as_string().count("\n")])
#elif part.get_main_type() == "text":
if part.get_main_type() == "text":
part_s.append(str(part.as_string().count("\n")))
if ext:
part_s.extend([md5.new(part.as_string()).digest(),
part.get('Content-Disposition'),
part.get('Content-Language')])
s.append(part_s)
if len(s) == 1:
return s[0]
return s
def body(self):
rfc822 = self.as_string()
bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
re.DOTALL + re.MULTILINE)
bmatch = bodyRE.search(rfc822)
return bmatch.group(2)
def headers(self):
rfc822 = self.as_string()
bodyRE = re.compile(r"\r?\n(\r?\n)(.*)",
re.DOTALL + re.MULTILINE)
bmatch = bodyRE.search(rfc822)
return rfc822[:bmatch.start(2)]
def on(self, date1, date2):
"contained within the date"
raise NotImplementedError
def before(self, date1, date2):
"before the date"
raise NotImplementedError
def since(self, date1, date2):
"within or after the date"
raise NotImplementedError
def string_contains(self, whole, sub):
return whole.find(sub) != -1
def matches(self, criteria):
"""Return True iff the messages matches the specified IMAP
criteria."""
match_tests = {"ALL" : [(True, True)],
"ANSWERED" : [(self.answered, True)],
"DELETED" : [(self.deleted, True)],
"DRAFT" : [(self.draft, True)],
"FLAGGED" : [(self.flagged, True)],
"NEW" : [(self.recent, True), (self.seen, False)],
"RECENT" : [(self.recent, True)],
"SEEN" : [(self.seen, True)],
"UNANSWERED" : [(self.answered, False)],
"UNDELETED" : [(self.deleted, False)],
"UNDRAFT" : [(self.draft, False)],
"UNFLAGGED" : [(self.flagged, False)],
"UNSEEN" : [(self.seen, False)],
"OLD" : [(self.recent, False)],
}
complex_tests = {"BCC" : (self.string_contains, self.get("Bcc")),
"SUBJECT" : (self.string_contains, self.get("Subject")),
"CC" : (self.string_contains, self.get("Cc")),
"BODY" : (self.string_contains, self.body()),
"TO" : (self.string_contains, self.get("To")),
"TEXT" : (self.string_contains, self.as_string()),
"FROM" : (self.string_contains, self.get("From")),
"SMALLER" : (operator.lt, len(self.as_string())),
"LARGER" : (operator.gt, len(self.as_string())),
"BEFORE" : (self.before, self.date),
"ON" : (self.on, self.date),
"SENTBEFORE" : (self.before, self.get("Date")),
"SENTON" : (self.on, self.get("Date")),
"SENTSINCE" : (self.since, self.get("Date")),
"SINCE" : (self.since, self.date),
}
result = True
test = None
header = None
header_field = None
for c in criteria:
if match_tests.has_key(c) and test is None and header is None:
for (test, result) in match_tests[c]:
result = result and (test == result)
elif complex_tests.has_key(c) and test is None and header is None:
test = complex_tests[c]
elif test is not None and header is None:
result = result and test[0](test[1], c)
test = None
elif c == "HEADER" and test is None:
# the only criteria that uses the next _two_ elements
header = c
elif test is None and header is not None and header_field is None:
header_field = c
elif header is not None and header_field is not None and test is None:
result = result and self.string_contains(self.get(header_field), c)
header = None
header_field = None
return result
"""
Still to do:
Messages with message sequence numbers
corresponding to the specified message sequence
number set
UID
Messages with unique identifiers corresponding to
the specified unique identifier set.
KEYWORD Messages with the specified keyword set.
UNKEYWORD
Messages that do not have the specified keyword
set.
NOT
Messages that do not match the specified search
key.
OR
Messages that match either search key.
"""
class IMAPFileMessageFactory(FileCorpus.FileMessageFactory):
'''MessageFactory for IMAPFileMessage objects'''
def create(self, key, directory):
'''Create a message object from a filename in a directory'''
return IMAPFileMessage(key, directory)
class IMAPMailbox(cred.perspective.Perspective):
__implements__ = (IMailbox,)
def __init__(self, name, identity_name, id):
cred.perspective.Perspective.__init__(self, name, identity_name)
self.UID_validity = id
self.listeners = []
def getUIDValidity(self):
"""Return the unique validity identifier for this mailbox."""
return self.UID_validity
def addListener(self, listener):
"""Add a mailbox change listener."""
self.listeners.append(listener)
def removeListener(self, listener):
"""Remove a mailbox change listener."""
self.listeners.remove(listener)
class SpambayesMailbox(IMAPMailbox):
def __init__(self, name, id, directory):
IMAPMailbox.__init__(self, name, "spambayes", id)
self.UID_validity = id
ensureDir(directory)
self.storage = FileCorpus.FileCorpus(IMAPFileMessageFactory(),
directory, r"[0123456789]*")
# UIDs are required to be strictly ascending.
if len(self.storage.keys()) == 0:
self.nextUID = 0
else:
self.nextUID = long(self.storage.keys()[-1]) + 1
# Calculate initial recent and unseen counts
# XXX Note that this will always end up with zero counts
# XXX until the flags are persisted.
self.unseen_count = 0
self.recent_count = 0
for msg in self.storage:
if not msg.seen:
self.unseen_count += 1
if msg.recent:
self.recent_count += 1
def getUIDNext(self, increase=False):
"""Return the likely UID for the next message added to this
mailbox."""
reply = str(self.nextUID)
if increase:
self.nextUID += 1
return reply
def getUID(self, message):
"""Return the UID of a message in the mailbox."""
# Note that IMAP messages are 1-based, our messages are 0-based
d = self.storage
return long(d.keys()[message - 1])
def getFlags(self):
"""Return the flags defined in this mailbox."""
return ["\\Answered", "\\Flagged", "\\Deleted", "\\Seen",
"\\Draft"]
def getMessageCount(self):
"""Return the number of messages in this mailbox."""
return len(self.storage.keys())
def getRecentCount(self):
"""Return the number of messages with the 'Recent' flag."""
return self.recent_count
def getUnseenCount(self):
"""Return the number of messages with the 'Unseen' flag."""
return self.unseen_count
def isWriteable(self):
"""Get the read/write status of the mailbox."""
return True
def destroy(self):
"""Called before this mailbox is deleted, permanently."""
# Our mailboxes cannot be deleted
raise NotImplementedError
def getHierarchicalDelimiter(self):
"""Get the character which delimits namespaces for in this
mailbox."""
return '.'
def requestStatus(self, names):
"""Return status information about this mailbox."""
answer = {}
for request in names:
request = request.upper()
if request == "MESSAGES":
answer[request] = self.getMessageCount()
elif request == "RECENT":
answer[request] = self.getRecentCount()
elif request == "UIDNEXT":
answer[request] = self.getUIDNext()
elif request == "UIDVALIDITY":
answer[request] = self.getUIDValidity()
elif request == "UNSEEN":
answer[request] = self.getUnseenCount()
return answer
def addMessage(self, message, flags=(), date=None):
"""Add the given message to this mailbox."""
msg = self.storage.makeMessage(self.getUIDNext(True))
msg.date = date
msg.setPayload(message.read())
self.storage.addMessage(msg)
self.store(MessageSet(long(msg.id), long(msg.id)), flags, 1, True)
msg.recent = True
msg.store()
self.recent_count += 1
self.unseen_count += 1
for listener in self.listeners:
listener.newMessages(self.getMessageCount(),
self.getRecentCount())
d = defer.Deferred()
reactor.callLater(0, d.callback, self.storage.keys().index(msg.id))
return d
def expunge(self):
"""Remove all messages flagged \\Deleted."""
deleted_messages = []
for msg in self.storage:
if msg.deleted:
if not msg.seen:
self.unseen_count -= 1
if msg.recent:
self.recent_count -= 1
deleted_messages.append(long(msg.id))
self.storage.removeMessage(msg)
if deleted_messages != []:
for listener in self.listeners:
listener.newMessages(self.getMessageCount(),
self.getRecentCount())
return deleted_messages
def search(self, query, uid):
"""Search for messages that meet the given query criteria.
@type query: C{list}
@param query: The search criteria
@rtype: C{list}
@return: A list of message sequence numbers or message UIDs which
match the search criteria.
"""
if self.getMessageCount() == 0:
return []
all_msgs = MessageSet(long(self.storage.keys()[0]),
long(self.storage.keys()[-1]))
matches = []
for id, msg in self._messagesIter(all_msgs, uid):
for q in query:
if msg.matches(q):
matches.append(id)
break
return matches
def _messagesIter(self, messages, uid):
if uid:
messages.last = long(self.storage.keys()[-1])
else:
messages.last = self.getMessageCount()
for id in messages:
if uid:
msg = self.storage.get(str(id))
else:
msg = self.storage.get(str(self.getUID(id)))
if msg is None:
# Non-existant message.
continue
msg.load()
yield (id, msg)
def fetch(self, messages, uid):
"""Retrieve one or more messages."""
return self._messagesIter(messages, uid)
def store(self, messages, flags, mode, uid):
"""Set the flags of one or more messages."""
stored_messages = {}
for id, msg in self._messagesIter(messages, uid):
if mode == 0:
msg.clear_flags()
value = True
elif mode == -1:
value = False
elif mode == 1:
value = True
for flag in flags:
if flag == '(' or flag == ')':
continue
if flag == "SEEN" and value == True and msg.seen == False:
self.unseen_count -= 1
if flag == "SEEN" and value == False and msg.seen == True:
self.unseen_count += 1
msg.set_flag(flag, value)
stored_messages[id] = msg.flags()
return stored_messages
class Trainer(object):
"""Listens to a given mailbox and trains new messages as spam or
ham."""
__implements__ = (IMailboxListener,)
def __init__(self, mailbox, asSpam):
self.mailbox = mailbox
self.asSpam = asSpam
def modeChanged(self, writeable):
# We don't care
pass
def flagsChanged(self, newFlags):
# We don't care
pass
def newMessages(self, exists, recent):
# We don't get passed the actual message, or the id of
# the message, of even the message number. We just get
# the total number of new/recent messages.
# However, this function should be called _every_ time
# that a new message appears, so we should be able to
# assume that the last message is the new one.
# (We ignore the recent count)
if exists is not None:
id = self.mailbox.getUID(exists)
msg = self.mailbox.storage[str(id)]
msg.train(state.bayes, self.asSpam)
class SpambayesAccount(MemoryAccount):
"""Account for Spambayes server."""
def __init__(self, id, ham, spam, unsure):
MemoryAccount.__init__(self, id)
self.mailboxes = {"SPAM" : spam,
"UNSURE" : unsure,
"TRAIN_AS_HAM" : ham}
def select(self, name, readwrite=1):
# 'INBOX' is a special case-insensitive name meaning the
# primary mailbox for the user; we interpret this as an alias
# for 'spam'
if name.upper() == "INBOX":
name = "SPAM"
return MemoryAccount.select(self, name, readwrite)
class SpambayesIMAPServer(IMAP4Server):
IDENT = "Spambayes IMAP Server IMAP4rev1 Ready"
def __init__(self, user_account):
IMAP4Server.__init__(self)
self.account = user_account
def authenticateLogin(self, user, passwd):
"""Lookup the account associated with the given parameters."""
if user == options["imapserver", "username"] and \
passwd == options["imapserver", "password"]:
return (IAccount, self.account, None)
raise cred.error.UnauthorizedLogin()
def connectionMade(self):
state.activeIMAPSessions += 1
state.totalIMAPSessions += 1
IMAP4Server.connectionMade(self)
def connectionLost(self, reason):
state.activeIMAPSessions -= 1
IMAP4Server.connectionLost(self, reason)
def do_CREATE(self, tag, args):
"""Creating new folders on the server is not permitted."""
self.sendNegativeResponse(tag, \
"Creation of new folders is not permitted")
auth_CREATE = (do_CREATE, IMAP4Server.arg_astring)
select_CREATE = auth_CREATE
def do_DELETE(self, tag, args):
"""Deleting folders on the server is not permitted."""
self.sendNegativeResponse(tag, \
"Deletion of folders is not permitted")
auth_DELETE = (do_DELETE, IMAP4Server.arg_astring)
select_DELETE = auth_DELETE
class OneParameterFactory(ServerFactory):
"""A factory that allows a single parameter to be passed to the created
protocol."""
def buildProtocol(self, addr):
"""Create an instance of a subclass of Protocol, passing a single
parameter."""
if self.parameter is not None:
p = self.protocol(self.parameter)
else:
p = self.protocol()
p.factory = self
return p
class MyBayesProxy(POP3ProxyBase):
"""Proxies between an email client and a POP3 server, redirecting
mail to the imap server as necessary. It acts on the following
POP3 commands:
o RETR:
o Adds the judgement header based on the raw headers and body
of the message.
"""
intercept_message = 'From: "Spambayes" \n' \
'Subject: Spambayes Intercept\n\nA message ' \
'was intercepted by Spambayes (it scored %s).\n' \
'\nYou may find it in the Spam or Unsure ' \
'folder.\n\n.\n'
def __init__(self, clientSocket, serverName, serverPort, spam, unsure):
POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'RETR': self.onRetr}
state.totalSessions += 1
state.activeSessions += 1
self.isClosed = False
self.spam_folder = spam
self.unsure_folder = unsure
def send(self, data):
"""Logs the data to the log file."""
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
try:
return POP3ProxyBase.send(self, data)
except socket.error:
self.close()
def recv(self, size):
"""Logs the data to the log file."""
data = POP3ProxyBase.recv(self, size)
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
return data
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
state.activeSessions -= 1
POP3ProxyBase.close(self)
def onTransaction(self, command, args, response):
"""Takes the raw request and response, and returns the
(possibly processed) response to pass back to the email client.
"""
handler = self.handlers.get(command, self.onUnknown)
return handler(command, args, response)
def onRetr(self, command, args, response):
"""Classifies the message. If the result is ham, then simply
pass it through. If the result is an unsure or spam, move it
to the appropriate IMAP folder."""
# Use '\n\r?\n' to detect the end of the headers in case of
# broken emails that don't use the proper line separators.
if re.search(r'\n\r?\n', response):
# Break off the first line, which will be '+OK'.
ok, messageText = response.split('\n', 1)
prob = state.bayes.spamprob(tokenize(messageText))
if prob < options["Categorization", "ham_cutoff"]:
# Return the +OK and the message with the header added.
state.numHams += 1
return ok + "\n" + messageText
elif prob > options["Categorization", "spam_cutoff"]:
dest_folder = self.spam_folder
state.numSpams += 1
else:
dest_folder = self.unsure_folder
state.numUnsure += 1
msg = StringIO.StringIO(messageText)
date = imaplib.Time2Internaldate(time.time())[1:-1]
dest_folder.addMessage(msg, (), date)
# We have to return something, because the client is expecting
# us to. We return a short message indicating that a message
# was intercepted.
return ok + "\n" + self.intercept_message % (prob,)
else:
# Must be an error response.
return response
def onUnknown(self, command, args, response):
"""Default handler; returns the server's response verbatim."""
return response
class MyBayesProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
MyBayesProxy objects to serve them.
"""
def __init__(self, serverName, serverPort, proxyPort, spam, unsure):
proxyArgs = (serverName, serverPort, spam, unsure)
Dibbler.Listener.__init__(self, proxyPort, MyBayesProxy, proxyArgs)
print 'Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class IMAPState(State):
def __init__(self):
State.__init__(self)
# Set up the extra statistics.
self.totalIMAPSessions = 0
self.activeIMAPSessions = 0
def buildServerStrings(self):
"""After the server details have been set up, this creates string
versions of the details, for display in the Status panel."""
self.serverPortString = str(self.imap_port)
# Also build proxy strings
State.buildServerStrings(self)
state = IMAPState()
# ===================================================================
# __main__ driver.
# ===================================================================
def setup():
# Setup app, boxes, trainers and account
proxyListeners = []
app = Application("SpambayesIMAPServer")
spam_box = SpambayesMailbox("Spam", 0, options["imapserver",
"spam_directory"])
unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver",
"unsure_directory"])
ham_train_box = SpambayesMailbox("TrainAsHam", 2,
options["imapserver", "ham_directory"])
spam_trainer = Trainer(spam_box, True)
ham_trainer = Trainer(ham_train_box, False)
spam_box.addListener(spam_trainer)
ham_train_box.addListener(ham_trainer)
user_account = SpambayesAccount(options["imapserver", "username"],
ham_train_box, spam_box, unsure_box)
# add IMAP4 server
f = OneParameterFactory()
f.protocol = SpambayesIMAPServer
f.parameter = user_account
state.imap_port = options["imapserver", "port"]
app.listenTCP(state.imap_port, f)
# add POP3 proxy
state.createWorkers()
for (server, serverPort), proxyPort in zip(state.servers,
state.proxyPorts):
listener = MyBayesProxyListener(server, serverPort, proxyPort,
spam_box, unsure_box)
proxyListeners.append(listener)
state.buildServerStrings()
# add web interface
httpServer = UserInterfaceServer(state.uiPort)
serverUI = ServerUserInterface(state, _recreateState)
httpServer.register(serverUI)
return app
def run():
# Read the arguments.
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbd:D:u:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
launchUI = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-b':
launchUI = True
elif opt == '-d': # dbm file
state.useDB = True
options["Storage", "persistent_storage_file"] = arg
elif opt == '-D': # pickle file
state.useDB = False
options["Storage", "persistent_storage_file"] = arg
elif opt == '-u':
state.uiPort = int(arg)
# Let the user know what they are using...
print get_version_string("IMAP Server")
print "and engine %s.\n" % (get_version_string(),)
# setup everything
app = setup()
# kick things off
thread.start_new_thread(Dibbler.run, (launchUI,))
app.run(save=False)
if __name__ == "__main__":
run()
--- NEW FILE: sb_server.py ---
#!/usr/bin/env python
"""A POP3 proxy that works with classifier.py, and adds a simple
X-Spambayes-Classification header (ham/spam/unsure) to each incoming
email. You point pop3proxy at your POP3 server, and configure your
email client to collect mail from the proxy then filter on the added
header. Usage:
pop3proxy.py [options] [ []]
is the name of your real POP3 server
is the port number of your real POP3 server, which
defaults to 110.
options:
-h : Displays this help message.
-d FILE : use the named DBM database file
-D FILE : the the named Pickle database file
-l port : proxy listens on this port number (default 110)
-u port : User interface listens on this port number
(default 8880; Browse http://localhost:8880/)
-b : Launch a web browser showing the user interface.
All command line arguments and switches take their default
values from the [pop3proxy] and [html_ui] sections of
bayescustomize.ini.
For safety, and to help debugging, the whole POP3 conversation is
written out to _pop3proxy.log for each run, if
options["globals", "verbose"] is True.
To make rebuilding the database easier, uploaded messages are appended
to _pop3proxyham.mbox and _pop3proxyspam.mbox.
"""
# This module is part of the spambayes project, which is Copyright 2002
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Richie Hindle "
__credits__ = "Tim Peters, Neale Pickett, Tim Stone, all the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
todo = """
Web training interface:
User interface improvements:
o Once the pieces are on separate pages, make the paste box bigger.
o Deployment: Windows executable? atlaxwin and ctypes? Or just
webbrowser?
o Save the stats (num classified, etc.) between sessions.
o "Reload database" button.
New features:
o Online manual.
o Links to project homepage, mailing list, etc.
o List of words with stats (it would have to be paged!) a la SpamSieve.
Code quality:
o Cope with the email client timing out and closing the connection.
Info:
o Slightly-wordy index page; intro paragraph for each page.
o In both stats and training results, report nham and nspam - warn if
they're very different (for some value of 'very').
o "Links" section (on homepage?) to project homepage, mailing list,
etc.
Gimmicks:
o Classify a web page given a URL.
o Graphs. Of something. Who cares what?
o NNTP proxy.
o Zoe...!
"""
import os, sys, re, errno, getopt, time, traceback, socket, cStringIO
from thread import start_new_thread
from email.Header import Header
import spambayes.message
from spambayes import Dibbler
from spambayes import storage
from spambayes.FileCorpus import FileCorpus, ExpiryFileCorpus
from spambayes.FileCorpus import FileMessageFactory, GzipFileMessageFactory
from spambayes.Options import options
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ProxyUI import ProxyUserInterface
from spambayes.Version import get_version_string
# Increase the stack size on MacOS X. Stolen from Lib/test/regrtest.py
if sys.platform == 'darwin':
try:
import resource
except ImportError:
pass
else:
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
newsoft = min(hard, max(soft, 1024*2048))
resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))
# number to add to STAT length for each msg to fudge for spambayes headers
HEADER_SIZE_FUDGE_FACTOR = 512
class ServerLineReader(Dibbler.BrighterAsyncChat):
"""An async socket that reads lines from a remote server and
simply calls a callback with the data. The BayesProxy object
can't connect to the real POP3 server and talk to it
synchronously, because that would block the process."""
lineCallback = None
def __init__(self, serverName, serverPort, lineCallback):
Dibbler.BrighterAsyncChat.__init__(self)
self.lineCallback = lineCallback
self.request = ''
self.set_terminator('\r\n')
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((serverName, serverPort))
except socket.error, e:
error = "Can't connect to %s:%d: %s" % (serverName, serverPort, e)
print >>sys.stderr, error
self.lineCallback('-ERR %s\r\n' % error)
self.lineCallback('') # "The socket's been closed."
self.close()
def collect_incoming_data(self, data):
self.request = self.request + data
def found_terminator(self):
self.lineCallback(self.request + '\r\n')
self.request = ''
def handle_close(self):
self.lineCallback('')
self.close()
class POP3ProxyBase(Dibbler.BrighterAsyncChat):
"""An async dispatcher that understands POP3 and proxies to a POP3
server, calling `self.onTransaction(request, response)` for each
transaction. Responses are not un-byte-stuffed before reaching
self.onTransaction() (they probably should be for a totally generic
POP3ProxyBase class, but BayesProxy doesn't need it and it would
mean re-stuffing them afterwards). self.onTransaction() should
return the response to pass back to the email client - the response
can be the verbatim response or a processed version of it. The
special command 'KILL' kills it (passing a 'QUIT' command to the
server).
"""
def __init__(self, clientSocket, serverName, serverPort):
Dibbler.BrighterAsyncChat.__init__(self, clientSocket)
self.request = ''
self.response = ''
self.set_terminator('\r\n')
self.command = '' # The POP3 command being processed...
self.args = [] # ...and its arguments
self.isClosing = False # Has the server closed the socket?
self.seenAllHeaders = False # For the current RETR or TOP
self.startTime = 0 # (ditto)
self.serverSocket = ServerLineReader(serverName, serverPort,
self.onServerLine)
def onTransaction(self, command, args, response):
"""Overide this. Takes the raw request and the response, and
returns the (possibly processed) response to pass back to the
email client.
"""
raise NotImplementedError
def onServerLine(self, line):
"""A line of response has been received from the POP3 server."""
isFirstLine = not self.response
self.response = self.response + line
# Is this the line that terminates a set of headers?
self.seenAllHeaders = self.seenAllHeaders or line in ['\r\n', '\n']
# Has the server closed its end of the socket?
if not line:
self.isClosing = True
# If we're not processing a command, just echo the response.
if not self.command:
self.push(self.response)
self.response = ''
# Time out after 30 seconds for message-retrieval commands if
# all the headers are down. The rest of the message will proxy
# straight through.
if self.command in ['TOP', 'RETR'] and \
self.seenAllHeaders and time.time() > self.startTime + 30:
self.onResponse()
self.response = ''
# If that's a complete response, handle it.
elif not self.isMultiline() or line == '.\r\n' or \
(isFirstLine and line.startswith('-ERR')):
self.onResponse()
self.response = ''
def isMultiline(self):
"""Returns True if the request should get a multiline
response (assuming the response is positive).
"""
if self.command in ['USER', 'PASS', 'APOP', 'QUIT',
'STAT', 'DELE', 'NOOP', 'RSET', 'KILL']:
return False
elif self.command in ['RETR', 'TOP', 'CAPA']:
return True
elif self.command in ['LIST', 'UIDL']:
return len(self.args) == 0
else:
# Assume that an unknown command will get a single-line
# response. This should work for errors and for POP-AUTH,
# and is harmless even for multiline responses - the first
# line will be passed to onTransaction and ignored, then the
# rest will be proxied straight through.
return False
def collect_incoming_data(self, data):
"""Asynchat override."""
self.request = self.request + data
def found_terminator(self):
"""Asynchat override."""
verb = self.request.strip().upper()
if verb == 'KILL':
self.socket.shutdown(2)
self.close()
raise SystemExit
elif verb == 'CRASH':
# For testing
x = 0
y = 1/x
self.serverSocket.push(self.request + '\r\n')
if self.request.strip() == '':
# Someone just hit the Enter key.
self.command = ''
self.args = []
else:
# A proper command.
splitCommand = self.request.strip().split()
self.command = splitCommand[0].upper()
self.args = splitCommand[1:]
self.startTime = time.time()
self.request = ''
def onResponse(self):
# We don't support pipelining, so if the command is CAPA and the
# response includes PIPELINING, hack out that line of the response.
if self.command == 'CAPA':
pipelineRE = r'(?im)^PIPELINING[^\n]*\n'
self.response = re.sub(pipelineRE, '', self.response)
# Pass the request and the raw response to the subclass and
# send back the cooked response.
if self.response:
cooked = self.onTransaction(self.command, self.args, self.response)
self.push(cooked)
# If onServerLine() decided that the server has closed its
# socket, close this one when the response has been sent.
if self.isClosing:
self.close_when_done()
# Reset.
self.command = ''
self.args = []
self.isClosing = False
self.seenAllHeaders = False
class BayesProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
BayesProxy objects to serve them.
"""
def __init__(self, serverName, serverPort, proxyPort):
proxyArgs = (serverName, serverPort)
Dibbler.Listener.__init__(self, proxyPort, BayesProxy, proxyArgs)
print 'Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class BayesProxy(POP3ProxyBase):
"""Proxies between an email client and a POP3 server, inserting
judgement headers. It acts on the following POP3 commands:
o STAT:
o Adds the size of all the judgement headers to the maildrop
size.
o LIST:
o With no message number: adds the size of an judgement header
to the message size for each message in the scan listing.
o With a message number: adds the size of an judgement header
to the message size.
o RETR:
o Adds the judgement header based on the raw headers and body
of the message.
o TOP:
o Adds the judgement header based on the raw headers and as
much of the body as the TOP command retrieves. This can
mean that the header might have a different value for
different calls to TOP, or for calls to TOP vs. calls to
RETR. I'm assuming that the email client will either not
make multiple calls, or will cope with the headers being
different.
o USER:
o Does no processing based on the USER command itself, but
expires any old messages in the three caches.
"""
def __init__(self, clientSocket, serverName, serverPort):
POP3ProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'STAT': self.onStat, 'LIST': self.onList,
'RETR': self.onRetr, 'TOP': self.onTop,
'USER': self.onUser}
state.totalSessions += 1
state.activeSessions += 1
self.isClosed = False
def send(self, data):
"""Logs the data to the log file."""
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
try:
return POP3ProxyBase.send(self, data)
except socket.error:
# The email client has closed the connection - 40tude Dialog
# does this immediately after issuing a QUIT command,
# without waiting for the response.
self.close()
def recv(self, size):
"""Logs the data to the log file."""
data = POP3ProxyBase.recv(self, size)
if options["globals", "verbose"]:
state.logFile.write(data)
state.logFile.flush()
return data
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
state.activeSessions -= 1
POP3ProxyBase.close(self)
def onTransaction(self, command, args, response):
"""Takes the raw request and response, and returns the
(possibly processed) response to pass back to the email client.
"""
handler = self.handlers.get(command, self.onUnknown)
return handler(command, args, response)
def onStat(self, command, args, response):
"""Adds the size of all the judgement headers to the maildrop
size."""
match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response)
if match:
count = int(match.group(1))
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR * count
return '+OK %d %d%s\r\n' % (count, size, match.group(3))
else:
return response
def onList(self, command, args, response):
"""Adds the size of an judgement header to the message
size(s)."""
if response.count('\r\n') > 1:
# Multiline: all lines but the first contain a message size.
lines = response.split('\r\n')
outputLines = [lines[0]]
for line in lines[1:]:
match = re.search(r'^(\d+)\s+(\d+)', line)
if match:
number = int(match.group(1))
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR
line = "%d %d" % (number, size)
outputLines.append(line)
return '\r\n'.join(outputLines)
else:
# Single line.
match = re.search(r'^\+OK\s+(\d+)\s+(\d+)(.*)\r\n', response)
if match:
messageNumber = match.group(1)
size = int(match.group(2)) + HEADER_SIZE_FUDGE_FACTOR
trailer = match.group(3)
return "+OK %s %s%s\r\n" % (messageNumber, size, trailer)
else:
return response
def onRetr(self, command, args, response):
"""Adds the judgement header based on the raw headers and body
of the message."""
# Use '\n\r?\n' to detect the end of the headers in case of
# broken emails that don't use the proper line separators.
if re.search(r'\n\r?\n', response):
# Remove the trailing .\r\n before passing to the email parser.
# Thanks to Scott Schlesier for this fix.
terminatingDotPresent = (response[-4:] == '\n.\r\n')
if terminatingDotPresent:
response = response[:-3]
# Break off the first line, which will be '+OK'.
ok, messageText = response.split('\n', 1)
try:
msg = spambayes.message.SBHeaderMessage()
msg.setPayload(messageText)
msg.setId(state.getNewMessageName())
# Now find the spam disposition and add the header.
(prob, clues) = state.bayes.spamprob(msg.asTokens(),\
evidence=True)
msg.addSBHeaders(prob, clues)
# Check for "RETR" or "TOP N 99999999" - fetchmail without
# the 'fetchall' option uses the latter to retrieve messages.
if (command == 'RETR' or
(command == 'TOP' and
len(args) == 2 and args[1] == '99999999')):
cls = msg.GetClassification()
if cls == options["Headers", "header_ham_string"]:
state.numHams += 1
elif cls == options["Headers", "header_spam_string"]:
state.numSpams += 1
else:
state.numUnsure += 1
# Suppress caching of "Precedence: bulk" or
# "Precedence: list" ham if the options say so.
isSuppressedBulkHam = \
(cls == options["Headers", "header_ham_string"] and
options["pop3proxy", "no_cache_bulk_ham"] and
msg.get('precedence') in ['bulk', 'list'])
# Suppress large messages if the options say so.
size_limit = options["pop3proxy",
"no_cache_large_messages"]
isTooBig = size_limit > 0 and \
len(messageText) > size_limit
# Cache the message. Don't pollute the cache with test
# messages or suppressed bulk ham.
if (not state.isTest and
options["pop3proxy", "cache_messages"] and
not isSuppressedBulkHam and not isTooBig):
# Write the message into the Unknown cache.
message = state.unknownCorpus.makeMessage(msg.getId())
message.setSubstance(msg.as_string())
state.unknownCorpus.addMessage(message)
# We'll return the message with the headers added. We take
# all the headers from the SBHeaderMessage, but take the body
# directly from the POP3 conversation, because the
# SBHeaderMessage might have "fixed" a partial message by
# appending a closing boundary separator. Remember we can
# be dealing with partial message here because of the timeout
# code in onServerLine.
headers = []
for name, value in msg.items():
header = "%s: %s" % (name, value)
headers.append(re.sub(r'\r?\n', '\r\n', header))
body = re.split(r'\n\r?\n', messageText, 1)[1]
messageText = "\r\n".join(headers) + "\r\n\r\n" + body
except:
# Something nasty happened while parsing or classifying -
# report the exception in a hand-appended header and recover.
# This is one case where an unqualified 'except' is OK, 'cos
# anything's better than destroying people's email...
stream = cStringIO.StringIO()
traceback.print_exc(None, stream)
details = stream.getvalue()
# Build the header. This will strip leading whitespace from
# the lines, so we add a leading dot to maintain indentation.
detailLines = details.strip().split('\n')
dottedDetails = '\n.'.join(detailLines)
headerName = 'X-Spambayes-Exception'
header = Header(dottedDetails, header_name=headerName)
# Insert the header, converting email.Header's '\n' line
# breaks to POP3's '\r\n'.
headers, body = re.split(r'\n\r?\n', messageText, 1)
header = re.sub(r'\r?\n', '\r\n', str(header))
headers += "\n%s: %s\r\n\r\n" % (headerName, header)
messageText = headers + body
# Print the exception and a traceback.
print >>sys.stderr, details
# Restore the +OK and the POP3 .\r\n terminator if there was one.
retval = ok + "\n" + messageText
if terminatingDotPresent:
retval += '.\r\n'
return retval
else:
# Must be an error response.
return response
def onTop(self, command, args, response):
"""Adds the judgement header based on the raw headers and as
much of the body as the TOP command retrieves."""
# Easy (but see the caveat in BayesProxy.__doc__).
return self.onRetr(command, args, response)
def onUser(self, command, args, response):
"""Spins off three separate threads that expires any old messages
in the three caches, but does not do any processing of the USER
command itself."""
start_new_thread(state.spamCorpus.removeExpiredMessages, ())
start_new_thread(state.hamCorpus.removeExpiredMessages, ())
start_new_thread(state.unknownCorpus.removeExpiredMessages, ())
return response
def onUnknown(self, command, args, response):
"""Default handler; returns the server's response verbatim."""
return response
# This keeps the global state of the module - the command-line options,
# statistics like how many mails have been classified, the handle of the
# log file, the Classifier and FileCorpus objects, and so on.
class State:
def __init__(self):
"""Initialises the State object that holds the state of the app.
The default settings are read from Options.py and bayescustomize.ini
and are then overridden by the command-line processing code in the
__main__ code below."""
# Open the log file.
if options["globals", "verbose"]:
self.logFile = open('_pop3proxy.log', 'wb', 0)
self.servers = []
self.proxyPorts = []
if options["pop3proxy", "remote_servers"]:
for server in options["pop3proxy", "remote_servers"]:
server = server.strip()
if server.find(':') > -1:
server, port = server.split(':', 1)
else:
port = '110'
self.servers.append((server, int(port)))
if options["pop3proxy", "listen_ports"]:
splitPorts = options["pop3proxy", "listen_ports"]
self.proxyPorts = map(_addressAndPort, splitPorts)
if len(self.servers) != len(self.proxyPorts):
print "pop3proxy_servers & pop3proxy_ports are different lengths!"
sys.exit()
# Load up the other settings from Option.py / bayescustomize.ini
self.useDB = options["Storage", "persistent_use_database"]
self.uiPort = options["html_ui", "port"]
self.launchUI = options["html_ui", "launch_browser"]
self.gzipCache = options["pop3proxy", "cache_use_gzip"]
self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"]
self.runTestServer = False
self.isTest = False
# Set up the statistics.
self.totalSessions = 0
self.activeSessions = 0
self.numSpams = 0
self.numHams = 0
self.numUnsure = 0
# Unique names for cached messages - see `getNewMessageName()` below.
self.lastBaseMessageName = ''
self.uniquifier = 2
def buildServerStrings(self):
"""After the server details have been set up, this creates string
versions of the details, for display in the Status panel."""
serverStrings = ["%s:%s" % (s, p) for s, p in self.servers]
self.serversString = ', '.join(serverStrings)
self.proxyPortsString = ', '.join(map(_addressPortStr, self.proxyPorts))
def createWorkers(self):
"""Using the options that were initialised in __init__ and then
possibly overridden by the driver code, create the Bayes object,
the Corpuses, the Trainers and so on."""
print "Loading database...",
if self.isTest:
self.useDB = True
options["Storage", "persistent_storage_file"] = \
'_pop3proxy_test.pickle' # This is never saved.
filename = options["Storage", "persistent_storage_file"]
filename = os.path.expanduser(filename)
self.bayes = storage.open_storage(filename, self.useDB)
# Don't set up the caches and training objects when running the self-test,
# so as not to clutter the filesystem.
if not self.isTest:
def ensureDir(dirname):
try:
os.mkdir(dirname)
except OSError, e:
if e.errno != errno.EEXIST:
raise
# Create/open the Corpuses. Use small cache sizes to avoid hogging
# lots of memory.
map(ensureDir, [options["pop3proxy", "spam_cache"],
options["pop3proxy", "ham_cache"],
options["pop3proxy", "unknown_cache"]])
if self.gzipCache:
factory = GzipFileMessageFactory()
else:
factory = FileMessageFactory()
age = options["pop3proxy", "cache_expiry_days"]*24*60*60
self.spamCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"spam_cache"],
'[0123456789\-]*',
cacheSize=20)
self.hamCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"ham_cache"],
'[0123456789\-]*',
cacheSize=20)
self.unknownCorpus = ExpiryFileCorpus(age, factory,
options["pop3proxy",
"unknown_cache"],
'[0123456789\-]*',
cacheSize=20)
# Given that (hopefully) users will get to the stage
# where they do not need to do any more regular training to
# be satisfied with spambayes' performance, we expire old
# messages from not only the trained corpora, but the unknown
# as well.
self.spamCorpus.removeExpiredMessages()
self.hamCorpus.removeExpiredMessages()
self.unknownCorpus.removeExpiredMessages()
# Create the Trainers.
self.spamTrainer = storage.SpamTrainer(self.bayes)
self.hamTrainer = storage.HamTrainer(self.bayes)
self.spamCorpus.addObserver(self.spamTrainer)
self.hamCorpus.addObserver(self.hamTrainer)
def getNewMessageName(self):
# The message name is the time it arrived, with a uniquifier
# appended if two arrive within one clock tick of each other.
messageName = "%10.10d" % long(time.time())
if messageName == self.lastBaseMessageName:
messageName = "%s-%d" % (messageName, self.uniquifier)
self.uniquifier += 1
else:
self.lastBaseMessageName = messageName
self.uniquifier = 2
return messageName
# Option-parsing helper functions
def _addressAndPort(s):
"""Decode a string representing a port to bind to, with optional address."""
s = s.strip()
if ':' in s:
addr, port = s.split(':')
return addr, int(port)
else:
return '', int(s)
def _addressPortStr((addr, port)):
"""Encode a string representing a port to bind to, with optional address."""
if not addr:
return str(port)
else:
return '%s:%d' % (addr, port)
state = State()
proxyListeners = []
def _createProxies(servers, proxyPorts):
"""Create BayesProxyListeners for all the given servers."""
for (server, serverPort), proxyPort in zip(servers, proxyPorts):
listener = BayesProxyListener(server, serverPort, proxyPort)
proxyListeners.append(listener)
def _recreateState():
global state
state = State()
# Close the existing listeners and create new ones. This won't
# affect any running proxies - once a listener has created a proxy,
# that proxy is then independent of it.
for proxy in proxyListeners:
proxy.close()
del proxyListeners[:]
prepare(state)
_createProxies(state.servers, state.proxyPorts)
return state
def main(servers, proxyPorts, uiPort, launchUI):
"""Runs the proxy forever or until a 'KILL' command is received or
someone hits Ctrl+Break."""
_createProxies(servers, proxyPorts)
httpServer = UserInterfaceServer(uiPort)
proxyUI = ProxyUserInterface(state, _recreateState)
httpServer.register(proxyUI)
Dibbler.run(launchBrowser=launchUI)
def prepare(state):
# Do whatever we've been asked to do...
state.createWorkers()
# Launch any SMTP proxies. Note that if the user hasn't specified any
# SMTP proxy information in their configuration, then nothing will
# happen.
import sb_smtpproxy
servers, proxyPorts = sb_smtpproxy.LoadServerInfo()
proxyListeners.extend(sb_smtpproxy.CreateProxies(servers, proxyPorts,
state))
# setup info for the web interface
state.buildServerStrings()
def start(state):
# kick everything off
main(state.servers, state.proxyPorts, state.uiPort, state.launchUI)
def stop(state):
# Shutdown as though through the web UI. This will save the DB, allow
# any open proxy connections to complete, etc.
from urllib2 import urlopen
from urllib import urlencode
urlopen('http://localhost:%d/save' % state.uiPort,
urlencode({'how': 'Save & shutdown'})).read()
# ===================================================================
# __main__ driver.
# ===================================================================
def run():
# Read the arguments.
try:
opts, args = getopt.getopt(sys.argv[1:], 'hbpsd:D:l:u:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
runSelfTest = False
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-b':
state.launchUI = True
elif opt == '-d': # dbm file
state.useDB = True
options["Storage", "persistent_storage_file"] = arg
elif opt == '-D': # pickle file
state.useDB = False
options["Storage", "persistent_storage_file"] = arg
elif opt == '-p': # dead option
print >>sys.stderr, "-p option is no longer supported, use -D\n"
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-l':
state.proxyPorts = [_addressAndPort(arg)]
elif opt == '-u':
state.uiPort = int(arg)
# Let the user know what they are using...
print get_version_string("POP3 Proxy")
print "and engine %s.\n" % (get_version_string(),)
prepare(state=state)
if 0 <= len(args) <= 2:
# Normal usage, with optional server name and port number.
if len(args) == 1:
state.servers = [(args[0], 110)]
elif len(args) == 2:
state.servers = [(args[0], int(args[1]))]
# Default to listening on port 110 for command-line-specified servers.
if len(args) > 0 and state.proxyPorts == []:
state.proxyPorts = [('', 110)]
start(state=state)
else:
print >>sys.stderr, __doc__
if __name__ == '__main__':
run()
--- NEW FILE: sb_smtpproxy.py ---
#!/usr/bin/env python
"""A SMTP proxy to train a Spambayes database.
You point SMTP Proxy at your SMTP server(s) and configure your email
client(s) to send mail through the proxy (i.e. usually this means you use
localhost as the outgoing server).
To setup, enter appropriate values in your Spambayes configuration file in
the "SMTP Proxy" section (in particular: "remote_servers", "listen_ports",
and "use_cached_message"). This configuration can also be carried out via
the web user interface offered by POP3 Proxy and IMAP Filter.
To use, simply forward/bounce mail that you wish to train to the
appropriate address (defaults to spambayes_spam@localhost and
spambayes_ham@localhost). All other mail is sent normally.
(Note that IMAP Filter and POP3 Proxy users should not execute this script;
launching of SMTP Proxy will be taken care of by those applicatons).
There are two main forms of operation. With both, mail to two
(user-configurable) email addresses is intercepted by the proxy (and is
*not* sent to the SMTP server) and used as training data for a Spambayes
database. All other mail is simply relayed to the SMTP server.
If the "use_cached_message" option is False, the proxy uses the message
sent as training data. This option is suitable for those not using
POP3 Proxy or IMAP Filter, or for those that are confident that their
mailer will forward/bounce messages in an unaltered form.
If the "use_cached_message" option is True, the proxy examines the message
for a unique spambayes identification number. It then tries to find this
message in the pop3proxy caches and on the imap servers. It then retrieves
the message from the cache/server and uses *this* as the training data.
This method is suitable for those using POP3 Proxy and/or IMAP Filter, and
avoids any potential problems with the mailer altering messages before
forwarding/bouncing them.
Usage:
smtpproxy [options]
note: option values with spaces must be enclosed in double quotes
options:
-d dbname : pickled training database filename
-D dbname : dbm training database filename
-h : help
-v : verbose mode
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "Tim Stone, all the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
todo = """
o It would be nice if spam/ham could be bulk forwarded to the proxy,
rather than one by one. This would require separating the different
messages and extracting the correct ids. Simply changing to find
*all* the ids in a message, rather than stopping after one *might*
work, but I don't really know. Richie Hindle suggested something along
these lines back in September '02.
o Suggestions?
Testing:
o Test with as many clients as possible to check that the
id is correctly extracted from the forwarded/bounced message.
MUA information:
A '*' in the Header column signifies that the smtpproxy can extract
the id from the headers only. A '*' in the Body column signifies that
the smtpproxy can extract the id from the body of the message, if it
is there.
Header Body
*** Windows 2000 MUAs ***
Eudora 5.2 Forward * *
Eudora 5.2 Redirect *
Netscape Messenger (4.7) Forward (inline) * *
Netscape Messenger (4.7) Forward (quoted) Plain *
Netscape Messenger (4.7) Forward (quoted) HTML *
Netscape Messenger (4.7) Forward (quoted) Plain & HTML *
Netscape Messenger (4.7) Forward (attachment) Plain * *
Netscape Messenger (4.7) Forward (attachment) HTML * *
Netscape Messenger (4.7) Forward (attachment) Plain & HTML * *
Outlook Express 6 Forward HTML (Base64) *
Outlook Express 6 Forward HTML (None) *
Outlook Express 6 Forward HTML (QP) *
Outlook Express 6 Forward Plain (Base64) *
Outlook Express 6 Forward Plain (None) *
Outlook Express 6 Forward Plain (QP) *
Outlook Express 6 Forward Plain (uuencoded) *
http://www.endymion.com/products/mailman Forward *
M2 (Opera Mailer 7.01) Forward *
M2 (Opera Mailer 7.01) Redirect * *
The Bat! 1.62i Forward (RFC Headers not visible) *
The Bat! 1.62i Forward (RFC Headers visible) * *
The Bat! 1.62i Redirect *
The Bat! 1.62i Alternative Forward * *
The Bat! 1.62i Custom Template * *
AllegroMail 2.5.0.2 Forward *
AllegroMail 2.5.0.2 Redirect *
PocoMail 2.6.3 Bounce *
PocoMail 2.6.3 Bounce *
Pegasus Mail 4.02 Forward (all headers option set) * *
Pegasus Mail 4.02 Forward (all headers option not set) *
Calypso 3 Forward *
Calypso 3 Redirect * *
Becky! 2.05.10 Forward *
Becky! 2.05.10 Redirect *
Becky! 2.05.10 Redirect as attachment * *
Mozilla Mail 1.2.1 Forward (attachment) * *
Mozilla Mail 1.2.1 Forward (inline, plain) *1 *
Mozilla Mail 1.2.1 Forward (inline, plain & html) *1 *
Mozilla Mail 1.2.1 Forward (inline, html) *1 *
*1 The header method will only work if auto-include original message
is set, and if view all headers is true.
"""
import string
import re
import socket
import asyncore
import asynchat
import getopt
import sys
import os
from spambayes import Dibbler
from spambayes import storage
from spambayes.message import sbheadermessage_from_string
from spambayes.tokenizer import textparts
from spambayes.tokenizer import try_to_repair_damaged_base64
from spambayes.Options import options
from sb_server import _addressPortStr, ServerLineReader
from sb_server import _addressAndPort
class SMTPProxyBase(Dibbler.BrighterAsyncChat):
"""An async dispatcher that understands SMTP and proxies to a SMTP
server, calling `self.onTransaction(command, args)` for each
transaction.
self.onTransaction() should return the command to pass to
the proxied server - the command can be the verbatim command or a
processed version of it. The special command 'KILL' kills it (passing
a 'QUIT' command to the server).
"""
def __init__(self, clientSocket, serverName, serverPort):
Dibbler.BrighterAsyncChat.__init__(self, clientSocket)
self.request = ''
self.set_terminator('\r\n')
self.command = '' # The SMTP command being processed...
self.args = '' # ...and its arguments
self.isClosing = False # Has the server closed the socket?
self.inData = False
self.data = ""
self.blockData = False
self.serverSocket = ServerLineReader(serverName, serverPort,
self.onServerLine)
def onTransaction(self, command, args):
"""Overide this. Takes the raw command and returns the (possibly
processed) command to pass to the email client."""
raise NotImplementedError
def onProcessData(self, data):
"""Overide this. Takes the raw data and returns the (possibly
processed) data to pass back to the email client."""
raise NotImplementedError
def onServerLine(self, line):
"""A line of response has been received from the SMTP server."""
# Has the server closed its end of the socket?
if not line:
self.isClosing = True
# We don't process the return, just echo the response.
self.push(line)
self.onResponse()
def collect_incoming_data(self, data):
"""Asynchat override."""
self.request = self.request + data
def found_terminator(self):
"""Asynchat override."""
verb = self.request.strip().upper()
if verb == 'KILL':
self.socket.shutdown(2)
self.close()
raise SystemExit
if self.request.strip() == '':
# Someone just hit the Enter key.
self.command = self.args = ''
else:
# A proper command.
if self.request[:10].upper() == "MAIL FROM:":
splitCommand = self.request.split(":", 1)
elif self.request[:8].upper() == "RCPT TO:":
splitCommand = self.request.split(":", 1)
else:
splitCommand = self.request.strip().split(None, 1)
self.command = splitCommand[0]
self.args = splitCommand[1:]
if self.inData == True:
self.data += self.request + '\r\n'
if self.request == ".":
self.inData = False
cooked = self.onProcessData(self.data)
self.data = ""
if self.blockData == False:
self.serverSocket.push(cooked)
else:
self.push("250 OK\r\n")
else:
cooked = self.onTransaction(self.command, self.args)
if cooked is not None:
self.serverSocket.push(cooked + '\r\n')
self.command = self.args = self.request = ''
def onResponse(self):
# If onServerLine() decided that the server has closed its
# socket, close this one when the response has been sent.
if self.isClosing:
self.close_when_done()
# Reset.
self.command = ''
self.args = ''
self.isClosing = False
class BayesSMTPProxyListener(Dibbler.Listener):
"""Listens for incoming email client connections and spins off
BayesSMTPProxy objects to serve them."""
def __init__(self, serverName, serverPort, proxyPort, trainer):
proxyArgs = (serverName, serverPort, trainer)
Dibbler.Listener.__init__(self, proxyPort, BayesSMTPProxy,
proxyArgs)
print 'SMTP Listener on port %s is proxying %s:%d' % \
(_addressPortStr(proxyPort), serverName, serverPort)
class BayesSMTPProxy(SMTPProxyBase):
"""Proxies between an email client and a SMTP server, inserting
judgement headers. It acts on the following SMTP commands:
o RCPT TO:
o Checks if the recipient address matches the key ham or spam
addresses, and if so notes this and does not forward a command to
the proxied server. In all other cases simply passes on the
verbatim command.
o DATA:
o Notes that we are in the data section. If (from the RCPT TO
information) we are receiving a ham/spam message to train on,
then do not forward the command on. Otherwise forward verbatim.
Any other commands are merely passed on verbatim to the server.
"""
def __init__(self, clientSocket, serverName, serverPort, trainer):
SMTPProxyBase.__init__(self, clientSocket, serverName, serverPort)
self.handlers = {'RCPT TO': self.onRcptTo, 'DATA': self.onData,
'MAIL FROM': self.onMailFrom}
self.trainer = trainer
self.isClosed = False
self.train_as_ham = False
self.train_as_spam = False
def send(self, data):
try:
return SMTPProxyBase.send(self, data)
except socket.error:
# The email client has closed the connection - 40tude Dialog
# does this immediately after issuing a QUIT command,
# without waiting for the response.
self.close()
def close(self):
# This can be called multiple times by async.
if not self.isClosed:
self.isClosed = True
SMTPProxyBase.close(self)
def stripAddress(self, address):
"""
Strip the leading & trailing <> from an address. Handy for
getting FROM: addresses.
"""
if '<' in address:
start = string.index(address, '<') + 1
end = string.index(address, '>')
return address[start:end]
else:
return address
def splitTo(self, address):
"""Return 'address' as undressed (host, fulladdress) tuple.
Handy for use with TO: addresses."""
start = string.index(address, '<') + 1
sep = string.index(address, '@') + 1
end = string.index(address, '>')
return (address[sep:end], address[start:end],)
def onTransaction(self, command, args):
handler = self.handlers.get(command.upper(), self.onUnknown)
return handler(command, args)
def onProcessData(self, data):
if self.train_as_spam:
self.trainer.train(data, True)
self.train_as_spam = False
return ""
elif self.train_as_ham:
self.trainer.train(data, False)
self.train_as_ham = False
return ""
return data
def onRcptTo(self, command, args):
toHost, toFull = self.splitTo(args[0])
if toFull == options["smtpproxy", "spam_address"]:
self.train_as_spam = True
self.train_as_ham = False
self.blockData = True
self.push("250 OK\r\n")
return None
elif toFull == options["smtpproxy", "ham_address"]:
self.train_as_ham = True
self.train_as_spam = False
self.blockData = True
self.push("250 OK\r\n")
return None
else:
self.blockData = False
return "%s:%s" % (command, ' '.join(args))
def onData(self, command, args):
self.inData = True
if self.train_as_ham == True or self.train_as_spam == True:
self.push("250 OK\r\n")
return None
rv = command
for arg in args:
rv += ' ' + arg
return rv
def onMailFrom(self, command, args):
"""Just like the default handler, but has the necessary colon."""
rv = "%s:%s" % (command, ' '.join(args))
return rv
def onUnknown(self, command, args):
"""Default handler."""
return self.request
class SMTPTrainer(object):
def __init__(self, classifier, state=None, imap=None):
self.classifier = classifier
self.state = state
self.imap = imap
def extractSpambayesID(self, data):
msg = message_from_string(data)
# The nicest MUA is one that forwards the header intact.
id = msg.get(options["Headers", "mailid_header_name"])
if id is not None:
return id
# Some MUAs will put it in the body somewhere, while others will
# put it in an attached MIME message.
id = self._find_id_in_text(msg.as_string())
if id is not None:
return id
# the message might be encoded
for part in textparts(msg):
# Decode, or take it as-is if decoding fails.
try:
text = part.get_payload(decode=True)
except:
text = part.get_payload(decode=False)
if text is not None:
text = try_to_repair_damaged_base64(text)
if text is not None:
id = self._find_id_in_text(text)
return id
return None
header_pattern = re.escape(options["Headers", "mailid_header_name"])
# A MUA might enclose the id in a table, thus the convoluted re pattern
# (Mozilla Mail does this with inline html)
header_pattern += r":\s*(\\s*\\s*)?([\d\-]+)"
header_re = re.compile(header_pattern)
def _find_id_in_text(self, text):
mo = self.header_re.search(text)
if mo is None:
return None
return mo.group(2)
def train(self, msg, isSpam):
try:
use_cached = options["smtpproxy", "use_cached_message"]
except KeyError:
use_cached = True
if use_cached:
id = self.extractSpambayesID(msg)
if id is None:
print "Could not extract id"
return
self.train_cached_message(id, isSpam)
# Otherwise, train on the forwarded/bounced message.
msg = sbheadermessage_from_string(msg)
id = msg.setIdFromPayload()
msg.delSBHeaders()
if id is None:
# No id, so we don't have any reliable method of remembering
# information about this message, so we just assume that it
# hasn't been trained before. We could generate some sort of
# checksum for the message and use that as an id (this would
# mean that we didn't need to store the id with the message)
# but that might be a little unreliable.
self.classifier.learn(msg.asTokens(), isSpam)
else:
if msg.GetTrained() == (not isSpam):
self.classifier.unlearn(msg.asTokens(), not isSpam)
msg.RememberTrained(None)
if msg.GetTrained() is None:
self.classifier.learn(msg.asTokens(), isSpam)
msg.RememberTrained(isSpam)
def train_cached_message(self, id, isSpam):
if not self.train_message_in_pop3proxy_cache(id, isSpam) and \
not self.train_message_on_imap_server(id, isSpam):
print "Could not find message (%s); perhaps it was " + \
"deleted from the POP3Proxy cache or the IMAP " + \
"server. This means that no training was done." % (id, )
def train_message_in_pop3proxy_cache(self, id, isSpam):
if self.state is None:
return False
sourceCorpus = None
for corpus in [self.state.unknownCorpus, self.state.hamCorpus,
self.state.spamCorpus]:
if corpus.get(id) is not None:
sourceCorpus = corpus
break
if corpus is None:
return False
if isSpam == True:
targetCorpus = self.state.spamCorpus
else:
targetCorpus = self.state.hamCorpus
targetCorpus.takeMessage(id, sourceCorpus)
self.classifier.store()
def train_message_on_imap_server(self, id, isSpam):
if self.imap is None:
return False
msg = self.imap.FindMessage(id)
if msg is None:
return False
if msg.GetTrained() == (not isSpam):
msg.get_substance()
msg.delSBHeaders()
self.classifier.unlearn(msg.asTokens(), not isSpam)
msg.RememberTrained(None)
if msg.GetTrained() is None:
msg.get_substance()
msg.delSBHeaders()
self.classifier.learn(msg.asTokens(), isSpam)
msg.RememberTrained(isSpam)
def LoadServerInfo():
# Load the proxy settings
servers = []
proxyPorts = []
if options["smtpproxy", "remote_servers"]:
for server in options["smtpproxy", "remote_servers"]:
server = server.strip()
if server.find(':') > -1:
server, port = server.split(':', 1)
else:
port = '25'
servers.append((server, int(port)))
if options["smtpproxy", "listen_ports"]:
splitPorts = options["smtpproxy", "listen_ports"]
proxyPorts = map(_addressAndPort, splitPorts)
if len(servers) != len(proxyPorts):
print "smtpproxy:remote_servers & smtpproxy:listen_ports are " + \
"different lengths!"
sys.exit()
return servers, proxyPorts
def CreateProxies(servers, proxyPorts, trainer):
"""Create BayesSMTPProxyListeners for all the given servers."""
# allow for old versions of pop3proxy
if not isinstance(trainer, SMTPTrainer):
trainer = SMTPTrainer(trainer.bayes, trainer)
proxyListeners = []
for (server, serverPort), proxyPort in zip(servers, proxyPorts):
listener = BayesSMTPProxyListener(server, serverPort, proxyPort,
trainer)
proxyListeners.append(listener)
return proxyListeners
def main():
"""Runs the proxy until a 'KILL' command is received or someone hits
Ctrl+Break."""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hvd:D:')
except getopt.error, msg:
print >>sys.stderr, str(msg) + '\n\n' + __doc__
sys.exit()
bdbname = options["Storage", "persistent_storage_file"]
useDBM = options["Storage", "persistent_use_database"]
for opt, arg in opts:
if opt == '-h':
print >>sys.stderr, __doc__
sys.exit()
elif opt == '-d':
useDBM = False
bdbname = arg
elif opt == '-D':
useDBM = True
bdbname = arg
elif opt == '-v':
options["globals", "verbose"] = True
bdbname = os.path.expanduser(bdbname)
if options["globals", "verbose"]:
print "Loading database %s..." % (bdbname),
if useDBM:
classifier = storage.DBDictClassifier(bdbname)
else:
classifier = storage.PickledClassifier(bdbname)
if options["globals", "verbose"]:
print "Done."
servers, proxyPorts = LoadServerInfo()
trainer = SMTPTrainer(classifier)
proxyListeners = CreateProxies(servers, proxyPorts, trainer)
Dibbler.run()
if __name__ == '__main__':
main()
--- NEW FILE: sb_unheader.py ---
#!/usr/bin/env python
"""
unheader.py: cleans headers from email messages. By default, this
removes SpamAssassin headers, specify a pattern with -p to supply
new headers to remove.
This is often needed because existing spamassassin headers can
provide killer spam clues, for all the wrong reasons.
"""
import re
import sys
import os
import glob
import mailbox
import email.Parser
import email.Message
import email.Generator
import getopt
def unheader(msg, pat):
pat = re.compile(pat)
for hdr in msg.keys():
if pat.match(hdr):
del msg[hdr]
# remain compatible with 2.2.1 - steal replace_header from 2.3 source
class Message(email.Message.Message):
def replace_header(self, _name, _value):
"""Replace a header.
Replace the first matching header found in the message, retaining
header order and case. If no matching header was found, a
KeyError is raised.
"""
_name = _name.lower()
for i, (k, v) in zip(range(len(self._headers)), self._headers):
if k.lower() == _name:
self._headers[i] = (k, _value)
break
else:
raise KeyError, _name
class Parser(email.Parser.HeaderParser):
def __init__(self):
email.Parser.Parser.__init__(self, Message)
def deSA(msg):
if msg['X-Spam-Status']:
if msg['X-Spam-Status'].startswith('Yes'):
pct = msg['X-Spam-Prev-Content-Type']
if pct:
msg['Content-Type'] = pct
pcte = msg['X-Spam-Prev-Content-Transfer-Encoding']
if pcte:
msg['Content-Transfer-Encoding'] = pcte
subj = re.sub(r'\*\*\*\*\*SPAM\*\*\*\*\* ', '',
msg['Subject'] or "")
if subj != msg["Subject"]:
msg.replace_header("Subject", subj)
body = msg.get_payload()
newbody = []
at_start = 1
for line in body.splitlines():
if at_start and line.startswith('SPAM: '):
continue
elif at_start:
at_start = 0
newbody.append(line)
msg.set_payload("\n".join(newbody))
unheader(msg, "X-Spam-")
def process_message(msg, dosa, pats):
if pats is not None:
unheader(msg, pats)
if dosa:
deSA(msg)
def process_mailbox(f, dosa=1, pats=None):
gen = email.Generator.Generator(sys.stdout, maxheaderlen=0)
for msg in mailbox.PortableUnixMailbox(f, Parser().parse):
process_message(msg, dosa, pats)
gen(msg, unixfrom=1)
def process_maildir(d, dosa=1, pats=None):
parser = Parser()
for fn in glob.glob(os.path.join(d, "cur", "*")):
print ("reading from %s..." % fn),
file = open(fn)
msg = parser.parse(file)
process_message(msg, dosa, pats)
tmpfn = os.path.join(d, "tmp", os.path.basename(fn))
tmpfile = open(tmpfn, "w")
print "writing to %s" % tmpfn
email.Generator.Generator(tmpfile, maxheaderlen=0)(msg, unixfrom=0)
os.rename(tmpfn, fn)
def usage():
print >> sys.stderr, "usage: unheader.py [ -p pat ... ] [ -s ] folder"
print >> sys.stderr, "-p pat gives a regex pattern used to eliminate unwanted headers"
print >> sys.stderr, "'-p pat' may be given multiple times"
print >> sys.stderr, "-s tells not to remove SpamAssassin headers"
print >> sys.stderr, "-d means treat folder as a Maildir"
def main(args):
headerpats = []
dosa = 1
ismbox = 1
try:
opts, args = getopt.getopt(args, "p:shd")
except getopt.GetoptError:
usage()
sys.exit(1)
else:
for opt, arg in opts:
if opt == "-h":
usage()
sys.exit(0)
elif opt == "-p":
headerpats.append(arg)
elif opt == "-s":
dosa = 0
elif opt == "-d":
ismbox = 0
pats = headerpats and "|".join(headerpats) or None
if len(args) != 1:
usage()
sys.exit(1)
if ismbox:
f = file(args[0])
process_mailbox(f, dosa, pats)
else:
process_maildir(args[0], dosa, pats)
if __name__ == "__main__":
main(sys.argv[1:])
--- NEW FILE: sb_upload.py ---
#!/usr/bin/env python
"""
Read a message or a mailbox file on standard input, upload it to a
web browser and write it to standard output.
usage: %(progname)s [-h] [-n] [-s server] [-p port] [-r N]
Options:
-h, --help - print help and exit
-n, --null - suppress writing to standard output (default %(null)s)
-s, --server= - provide alternate web server (default %(server)s)
-p, --port= - provide alternate server port (default %(port)s)
-r, --prob= - feed the message to the trainer w/ prob N [0.0...1.0]
"""
import sys
import httplib
import mimetypes
import getopt
import random
from spambayes.Options import options
progname = sys.argv[0]
__author__ = "Skip Montanaro "
__credits__ = "Spambayes gang, Wade Leftwich"
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
# appropriated verbatim from a recipe by Wade Leftwich in the Python
# Cookbook: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
def post_multipart(host, selector, fields, files):
"""
Post fields and files to an http host as multipart/form-data. fields is
a sequence of (name, value) elements for regular form fields. files is
a sequence of (name, filename, value) elements for data to be uploaded
as files. Return the server's response page.
"""
content_type, body = encode_multipart_formdata(fields, files)
h = httplib.HTTP(host)
h.putrequest('POST', selector)
h.putheader('content-type', content_type)
h.putheader('content-length', str(len(body)))
h.endheaders()
h.send(body)
errcode, errmsg, headers = h.getreply()
return h.file.read()
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be
uploaded as files. Return (content_type, body) ready for httplib.HTTP
instance
"""
BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
def usage(*args):
defaults = {}
for d in args:
defaults.update(d)
print __doc__ % defaults
def main(argv):
null = False
server = "localhost"
port = options["html_ui", "port"]
prob = 1.0
try:
opts, args = getopt.getopt(argv, "hns:p:r:",
["help", "null", "server=", "port=",
"prob="])
except getopt.error:
usage(globals(), locals())
sys.exit(1)
for opt, arg in opts:
if opt in ("-h", "--help"):
usage(globals(), locals())
sys.exit(0)
elif opt in ("-n", "--null"):
null = True
elif opt in ("-s", "--server"):
server = arg
elif opt in ("-p", "--port"):
port = int(arg)
elif opt in ("-r", "--prob"):
n = float(arg)
if n < 0.0 or n > 1.0:
usage(globals(), locals())
sys.exit(1)
prob = n
if args:
usage(globals(), locals())
sys.exit(1)
data = sys.stdin.read()
sys.stdout.write(data)
if random.random() < prob:
try:
post_multipart("%s:%d"%(server,port), "/upload", [],
[('file', 'message.dat', data)])
except:
# not an error if the server isn't responding
pass
if __name__ == "__main__":
main(sys.argv[1:])
--- NEW FILE: sb_xmlrpcserver.py ---
#! /usr/bin/env python
# A server version of hammie.py
"""Usage: %(program)s [options] IP:PORT
Where:
-h
show usage and exit
-p FILE
use file as the persistent store. loads data from this file if it
exists, and saves data to this file at the end. Default: %(DEFAULTDB)s
-d
use the DBM store instead of cPickle. The file is larger and
creating it is slower, but checking against it is much faster,
especially for large word databases.
IP
IP address to bind (use 0.0.0.0 to listen on all IPs of this machine)
PORT
Port number to listen to.
"""
import SimpleXMLRPCServer
import getopt
import sys
import traceback
import xmlrpclib
from spambayes import hammie
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
program = sys.argv[0] # For usage(); referenced by docstring above
# Default DB path
DEFAULTDB = hammie.DEFAULTDB
class XMLHammie(hammie.Hammie):
def score(self, msg, *extra):
try:
msg = msg.data
except AttributeError:
pass
return xmlrpclib.Binary(hammie.Hammie.score(self, msg, *extra))
def filter(self, msg, *extra):
try:
msg = msg.data
except AttributeError:
pass
return xmlrpclib.Binary(hammie.Hammie.filter(self, msg, *extra))
class HammieHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
def do_POST(self):
"""Handles the HTTP POST request.
Attempts to interpret all HTTP POST requests as XML-RPC calls,
which are forwarded to the _dispatch method for handling.
This one also prints out tracebacks, to help me debug :)
"""
try:
# get arguments
data = self.rfile.read(int(self.headers["content-length"]))
params, method = xmlrpclib.loads(data)
# generate response
try:
response = self._dispatch(method, params)
# wrap response in a singleton tuple
response = (response,)
except:
traceback.print_exc()
# report exception back to server
response = xmlrpclib.dumps(
xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
)
else:
response = xmlrpclib.dumps(response, methodresponse=1)
except:
# internal error, report as HTTP server error
traceback.print_exc()
print `data`
self.send_response(500)
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
def usage(code, msg=''):
"""Print usage message and sys.exit(code)."""
if msg:
print >> sys.stderr, msg
print >> sys.stderr
print >> sys.stderr, __doc__ % globals()
sys.exit(code)
def main():
"""Main program; parse options and go."""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hdp:')
except getopt.error, msg:
usage(2, msg)
pck = DEFAULTDB
usedb = False
for opt, arg in opts:
if opt == '-h':
usage(0)
elif opt == '-p':
pck = arg
elif opt == "-d":
usedb = True
if len(args) != 1:
usage(2, "IP:PORT not specified")
ip, port = args[0].split(":")
port = int(port)
bayes = hammie.createbayes(pck, usedb)
h = XMLHammie(bayes)
server = SimpleXMLRPCServer.SimpleXMLRPCServer((ip, port), HammieHandler)
server.register_instance(h)
server.serve_forever()
if __name__ == "__main__":
main()
--- sb-client.py DELETED ---
--- sb-dbexpimp.py DELETED ---
--- sb-filter.py DELETED ---
--- sb-imapfilter.py DELETED ---
--- sb-mailsort.py DELETED ---
--- sb-mboxtrain.py DELETED ---
--- sb-notesfilter.py DELETED ---
--- sb-pop3dnd.py DELETED ---
--- sb-server.py DELETED ---
--- sb-smtpproxy.py DELETED ---
--- sb-unheader.py DELETED ---
--- sb-upload.py DELETED ---
--- sb-xmlrpcserver.py DELETED ---
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:25:29 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:25:35 2003
Subject: [Spambayes-checkins] spambayes/spambayes Histogram.py, 1.2,
1.3 Tester.py, 1.2, 1.3 message.py, 1.34, 1.35
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv15577/spambayes
Modified Files:
Histogram.py Tester.py message.py
Log Message:
Missed some options cleanup.
sb-server, sb-smtpproxy, sb-imapfilter, timtest and timcv all appear to work now. This should at least mean that the core does.
Index: Histogram.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Histogram.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** Histogram.py 14 Jan 2003 05:38:20 -0000 1.2
--- Histogram.py 5 Sep 2003 01:25:27 -0000 1.3
***************
*** 18,22 ****
# Note: nbuckets can be passed for backward compatibility. The
# display() method can be passed a different nbuckets value.
! def __init__(self, nbuckets=options.nbuckets, lo=0.0, hi=100.0):
self.lo, self.hi = lo, hi
self.nbuckets = nbuckets
--- 18,23 ----
# Note: nbuckets can be passed for backward compatibility. The
# display() method can be passed a different nbuckets value.
! def __init__(self, nbuckets=options["TestDriver", "nbuckets"],
! lo=0.0, hi=100.0):
self.lo, self.hi = lo, hi
self.nbuckets = nbuckets
***************
*** 76,80 ****
# Compute percentiles.
self.pct = pct = []
! for p in options.percentiles:
assert 0.0 <= p <= 100.0
# In going from data index 0 to index n-1, we move n-1 times.
--- 77,81 ----
# Compute percentiles.
self.pct = pct = []
! for p in options["TestDriver", "percentiles"]:
assert 0.0 <= p <= 100.0
# In going from data index 0 to index n-1, we move n-1 times.
Index: Tester.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Tester.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** Tester.py 14 Jan 2003 05:38:20 -0000 1.2
--- Tester.py 5 Sep 2003 01:25:27 -0000 1.3
***************
*** 93,98 ****
if callback:
callback(example, prob)
! is_ham_guessed = prob < options.ham_cutoff
! is_spam_guessed = prob >= options.spam_cutoff
if is_spam:
self.nspam_tested += 1
--- 93,98 ----
if callback:
callback(example, prob)
! is_ham_guessed = prob < options["Categorization", "ham_cutoff"]
! is_spam_guessed = prob >= options["Categorization", "spam_cutoff"]
if is_spam:
self.nspam_tested += 1
***************
*** 152,156 ****
>>> from spambayes.classifier import Bayes
>>> from spambayes.Options import options
! >>> options.ham_cutoff = options.spam_cutoff = 0.5
>>> good1 = _Example('', ['a', 'b', 'c'])
--- 152,156 ----
>>> from spambayes.classifier import Bayes
>>> from spambayes.Options import options
! >>> options["Categorization", "ham_cutoff"] = options["Categorization", "spam_cutoff"] = 0.5
>>> good1 = _Example('', ['a', 'b', 'c'])
Index: message.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -d -r1.34 -r1.35
*** message.py 26 Aug 2003 06:49:56 -0000 1.34
--- message.py 5 Sep 2003 01:25:27 -0000 1.35
***************
*** 266,270 ****
def setIdFromPayload(self):
try:
! self.setId(self[options['pop3proxy','mailid_header_name']])
except ValueError:
return None
--- 266,270 ----
def setIdFromPayload(self):
try:
! self.setId(self[options['Headers','mailid_header_name']])
except ValueError:
return None
***************
*** 336,340 ****
if options['Headers','add_unique_id']:
! self[options['pop3proxy','mailid_header_name']] = self.id
def delSBHeaders(self):
--- 336,340 ----
if options['Headers','add_unique_id']:
! self[options['Headers','mailid_header_name']] = self.id
def delSBHeaders(self):
From anadelonbrin at users.sourceforge.net Thu Sep 4 19:25:29 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 4 21:25:43 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv15577/scripts
Modified Files:
sb_imapfilter.py
Log Message:
Missed some options cleanup.
sb-server, sb-smtpproxy, sb-imapfilter, timtest and timcv all appear to work now. This should at least mean that the core does.
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_imapfilter.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_imapfilter.py 5 Sep 2003 01:25:27 -0000 1.2
***************
*** 581,588 ****
cls = msg.GetClassification()
! if cls == options["Hammie", "header_ham_string"]:
# we leave ham alone
count["ham"] += 1
! elif cls == options["Hammie", "header_spam_string"]:
msg.MoveTo(spamfolder)
count["spam"] += 1
--- 581,588 ----
cls = msg.GetClassification()
! if cls == options["Headers", "header_ham_string"]:
# we leave ham alone
count["ham"] += 1
! elif cls == options["Headers", "header_spam_string"]:
msg.MoveTo(spamfolder)
count["spam"] += 1
From mhammond at users.sourceforge.net Thu Sep 4 20:09:44 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 4 22:09:55 2003
Subject: [Spambayes-checkins]
spambayes/Outlook2000/dialogs __init__.py, 1.10, 1.11
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv22796
Modified Files:
__init__.py
Log Message:
rc_size was not set when rc file did not exist.
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/__init__.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** __init__.py 26 Aug 2003 21:29:08 -0000 1.10
--- __init__.py 5 Sep 2003 02:09:42 -0000 1.11
***************
*** 31,35 ****
rc_size = stat_data[stat.ST_SIZE]
except OSError:
! rc_mtime = None
if rc_mtime!=mtime or rc_size!=size:
# Need to generate the dialog.
--- 31,35 ----
rc_size = stat_data[stat.ST_SIZE]
except OSError:
! rc_mtime = rc_size = None
if rc_mtime!=mtime or rc_size!=size:
# Need to generate the dialog.
From richiehindle at users.sourceforge.net Thu Sep 4 01:14:59 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Thu Sep 4 22:40:49 2003
Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.16,1.17
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv991
Modified Files:
WHAT_IS_NEW.txt
Log Message:
Draw more attention to some incompatible options changes.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** WHAT_IS_NEW.txt 2 Sep 2003 01:31:03 -0000 1.16
--- WHAT_IS_NEW.txt 4 Sep 2003 07:14:57 -0000 1.17
***************
*** 13,16 ****
--- 13,30 ----
======================
+ --------------------------
+ ** Incompatible changes **
+ --------------------------
+
+ The values taken by some options have changed, so if you're upgrading from a
+ previous version, you may need to update your configuration file (.spambayesrc
+ or bayescustomize.ini)
+
+ o allow_remote_connections now takes a list of allowed IP addresses, or the
+ word 'localhost', or an asterisk (meaning all connections are accepted).
+ o notate_to and notate_subject now take a comma-separated list of one or
+ more of 'spam', 'ham' and 'unsure', allowing you to control which classes
+ of message are notated.
+
Outlook Plugin
--------------
From mhammond at users.sourceforge.net Fri Sep 5 00:51:03 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 02:51:07 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
dialogs.rc, 1.36, 1.37 dialogs.h, 1.19, 1.20
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv30645
Modified Files:
dialogs.rc dialogs.h
Log Message:
Reference the folder bitmap in the resource file.
Index: dialogs.rc
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v
retrieving revision 1.36
retrieving revision 1.37
diff -C2 -d -r1.36 -r1.37
*** dialogs.rc 29 Aug 2003 06:23:51 -0000 1.36
--- dialogs.rc 5 Sep 2003 06:51:00 -0000 1.37
***************
*** 684,687 ****
--- 684,694 ----
+ /////////////////////////////////////////////////////////////////////////////
+ //
+ // Bitmap
+ //
+
+ IDB_FOLDERS BITMAP "folders.bmp"
+
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
Index: dialogs.h
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.h,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** dialogs.h 27 Aug 2003 00:07:07 -0000 1.19
--- dialogs.h 5 Sep 2003 06:51:00 -0000 1.20
***************
*** 9,13 ****
#define IDD_FOLDER_SELECTOR 105
#define IDD_ADVANCED 106
- #define IDB_BITMAP1 107
#define IDD_GENERAL 108
#define IDD_FILTER_SPAM 110
--- 9,12 ----
***************
*** 26,29 ****
--- 25,29 ----
#define IDD_WIZARD_FINISHED_TRAIN_LATER 124
#define IDB_SBWIZLOGO 125
+ #define IDB_FOLDERS 127
#define IDC_PROGRESS 1000
#define IDC_PROGRESS_TEXT 1001
***************
*** 41,45 ****
#define IDC_BUT_FILTER_ENABLE 1013
#define IDC_FILTER_STATUS 1014
- #define IDC_BUT_FILTER_NOW 1015
#define IDC_BUT_FILTER_DEFINE 1016
#define IDC_BUT_ABOUT 1017
--- 41,44 ----
***************
*** 48,56 ****
#define IDC_BUT_UNREAD 1020
#define IDC_BUT_UNSEEN 1021
- #define IDC_BUT_FILTERNOW 1022
#define IDC_SLIDER_CERTAIN 1023
#define IDC_EDIT_CERTAIN 1024
#define IDC_ACTION_CERTAIN 1025
- #define IDC_TOFOLDER_CERTAIN 1026
#define IDC_FOLDER_CERTAIN 1027
#define IDC_BROWSE_CERTAIN 1028
--- 47,53 ----
***************
*** 58,62 ****
#define IDC_EDIT_UNSURE 1030
#define IDC_ACTION_UNSURE 1031
- #define IDC_TOFOLDER_UNSURE 1032
#define IDC_FOLDER_UNSURE 1033
#define IDC_BROWSE_UNSURE 1034
--- 55,58 ----
***************
*** 71,80 ****
#define IDC_STATUS1 1043
#define IDC_STATUS2 1044
- #define IDC_BUT_CLEARALL2 1045
#define IDC_BUT_NEW 1046
#define IDC_MARK_SPAM_AS_READ 1047
#define IDC_SAVE_SPAM_SCORE 1048
#define IDC_MARK_UNSURE_AS_READ 1051
- #define IDC_ADVANCED_BTN 1055
#define IDC_DELAY1_SLIDER 1056
#define IDC_DELAY1_TEXT 1057
--- 67,74 ----
***************
*** 85,90 ****
#define IDB_SBLOGO 1062
#define IDC_LOGO_GRAPHIC 1063
- #define IDC_USE_DELAY1 1064
- #define IDC_USE_DELAY2 1065
#define IDC_TAB 1068
#define IDC_BACK_BTN 1069
--- 79,82 ----
***************
*** 95,102 ****
#define IDC_DEL_SPAM_RS 1074
#define IDC_RECOVER_RS 1075
- #define IDC_HIDDEN 1076
#define IDC_FORWARD_BTN 1077
#define IDC_PAGE_PLACEHOLDER 1078
- #define IDC_SHOW_DATA_FOLDER2 1079
#define IDC_BUT_SHOW_DIAGNOSTICS 1080
#define IDC_BUT_PREPARATION 1081
--- 87,92 ----
***************
*** 112,116 ****
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
! #define _APS_NEXT_RESOURCE_VALUE 126
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1094
--- 102,106 ----
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
! #define _APS_NEXT_RESOURCE_VALUE 128
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1094
From mhammond at users.sourceforge.net Fri Sep 5 00:53:26 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 02:53:30 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
__init__.py, 1.2, 1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv30906
Modified Files:
__init__.py
Log Message:
Add GetImageParamsFromBitmapID() to centralize bitmap location logic,
hopefully taking into account a py2exe future.
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/__init__.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** __init__.py 10 Aug 2003 07:26:50 -0000 1.2
--- __init__.py 5 Sep 2003 06:53:24 -0000 1.3
***************
*** 1 ****
! # Empty file to designate this directory as a module.
--- 1,22 ----
! # Package that manages and defines dialog resources
!
! def GetImageParamsFromBitmapID(rc_parser, bmpid):
! import os, sys
! import win32gui, win32con, win32api
! if type(bmpid)==type(0):
! bmpid = rc_parser.names[bmpid]
! # For both binary and source versions, we currently load from files.
! # In future py2exe built binary versions we will be able to load the
! # bitmaps directly from our DLL.
! filename = rc_parser.bitmaps[bmpid]
! if hasattr(sys, "frozen"):
! # bitmap in the app/images directory
! # dont have manager available :(
! dll_filename = win32api.GetModuleFileName(sys.frozendllhandle)
! app_dir = os.path.dirname(dll_filename)
! filename = os.path.join(app_dir, "images", filename)
! else:
! if not os.path.isabs(filename):
! # In this directory
! filename = os.path.join( os.path.dirname( __file__ ), filename)
! return 0, filename, win32con.LR_LOADFROMFILE
From mhammond at users.sourceforge.net Fri Sep 5 00:54:25 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 02:54:29 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
FolderSelector.py, 1.26, 1.27 processors.py, 1.11, 1.12
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv31065
Modified Files:
FolderSelector.py processors.py
Log Message:
Centralize bitmap locating code (the folder selector didn't work in the
binary build)
Index: FolderSelector.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** FolderSelector.py 27 Aug 2003 06:53:27 -0000 1.26
--- FolderSelector.py 5 Sep 2003 06:54:23 -0000 1.27
***************
*** 426,437 ****
win32gui.SendMessage(child, win32con.BM_SETCHECK, self.checkbox_state)
self.list = self.GetDlgItem("IDC_LIST_FOLDERS")
!
! fname = os.path.join(os.path.dirname(__file__), "resources/folders.bmp")
bitmapMask = win32api.RGB(0,0,255)
! self.imageList = win32gui.ImageList_LoadImage(0, fname,
16, 0,
bitmapMask,
win32con.IMAGE_BITMAP,
! win32con.LR_LOADFROMFILE)
win32gui.SendMessage( self.list,
commctrl.TVM_SETIMAGELIST,
--- 426,438 ----
win32gui.SendMessage(child, win32con.BM_SETCHECK, self.checkbox_state)
self.list = self.GetDlgItem("IDC_LIST_FOLDERS")
! import resources
! mod_handle, mod_bmp, extra_flags = \
! resources.GetImageParamsFromBitmapID(self.dialog_parser, "IDB_FOLDERS")
bitmapMask = win32api.RGB(0,0,255)
! self.imageList = win32gui.ImageList_LoadImage(mod_handle, mod_bmp,
16, 0,
bitmapMask,
win32con.IMAGE_BITMAP,
! extra_flags)
win32gui.SendMessage( self.list,
commctrl.TVM_SETIMAGELIST,
***************
*** 466,470 ****
timer.kill_timer(self.timer_id)
self.item_map = None
! win32gui.ImageList_Destroy(self.imageList)
FolderSelector_Parent.OnDestroy(self, hwnd, msg, wparam, lparam)
--- 467,472 ----
timer.kill_timer(self.timer_id)
self.item_map = None
! if self.imageList:
! win32gui.ImageList_Destroy(self.imageList)
FolderSelector_Parent.OnDestroy(self, hwnd, msg, wparam, lparam)
Index: processors.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/processors.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** processors.py 27 Aug 2003 04:09:28 -0000 1.11
--- processors.py 5 Sep 2003 06:54:23 -0000 1.12
***************
*** 53,73 ****
def Init(self):
rcp = self.window.manager.dialog_parser;
! text = win32gui.GetWindowText(self.GetControl())
! name = rcp.names[int(text)]
! if bitmap_handles.has_key(name):
! handle = bitmap_handles[name]
else:
! filename = rcp.bitmaps[name]
! import os, sys
! if hasattr(sys, "frozen"):
! # bitmap in the app/images directory
! filename = os.path.join(self.window.manager.application_directory,
! "images", filename)
! else:
! if not os.path.isabs(filename):
! filename = os.path.join( os.path.dirname( __file__ ), "resources", filename)
! handle = win32gui.LoadImage(0, filename, win32con.IMAGE_BITMAP,0,0,
! win32con.LR_COLOR|win32con.LR_LOADFROMFILE|win32con.LR_SHARED)
! bitmap_handles[name] = handle
win32gui.SendMessage(self.GetControl(), win32con.STM_SETIMAGE, win32con.IMAGE_BITMAP, handle)
--- 53,67 ----
def Init(self):
rcp = self.window.manager.dialog_parser;
! bmp_id = int(win32gui.GetWindowText(self.GetControl()))
!
! if bitmap_handles.has_key(bmp_id):
! handle = bitmap_handles[bmp_id]
else:
! import resources
! mod_handle, mod_bmp, extra_flags = resources.GetImageParamsFromBitmapID(rcp, bmp_id)
! load_flags = extra_flags|win32con.LR_COLOR|win32con.LR_SHARED
! handle = win32gui.LoadImage(mod_handle, mod_bmp,
! win32con.IMAGE_BITMAP,0,0,load_flags)
! bitmap_handles[bmp_id] = handle
win32gui.SendMessage(self.GetControl(), win32con.STM_SETIMAGE, win32con.IMAGE_BITMAP, handle)
From richiehindle at users.sourceforge.net Fri Sep 5 01:06:52 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Fri Sep 5 03:06:55 2003
Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv551
Modified Files:
README-DEVEL.txt
Log Message:
Added instructions on how to make a source release.
Index: README-DEVEL.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** README-DEVEL.txt 25 Aug 2003 01:36:41 -0000 1.1
--- README-DEVEL.txt 5 Sep 2003 07:06:50 -0000 1.2
***************
*** 319,320 ****
--- 319,353 ----
factor of 10 or more across training sets, and even within runs against
a single training set), but harder to spot using N-fold c-v.
+
+
+ Making a source release
+ =======================
+
+ Source releases are built with distutils. Here's how I (Richie) have been
+ building them. I do this on a Windows box, partly so that the zip release
+ can have Windows line endings without needing to run a conversion script.
+ I don't think that's actually necessary, because everything would work on
+ Windows even with Unix line endings, but you couldn't load the files into
+ Notepad and sometimes it's convenient to do so. End users might not even
+ have any other text editor, so it make things like the README unREADable.
+ 8-)
+
+ o Checkout the 'spambayes' module twice, once with Windows line endings
+ and once with Unix line endings (I use WinCVS for this, using "Admin /
+ Preferences / Globals / Checkout text files with the Unix LF".
+ o Change spambayes/__init__.py to contain the new version number.
+ o In the Windows checkout, run "python setup.py sdist --formats zip"
+ o In the Unix checkout, run "python setup.py sdist --formats gztar"
+ o Take the resulting spambayes-1.0a5.zip and spambayes-1.0a5.tar.gz, and
+ test the former on Windows (ideally in a freshly-installed Python
+ environment; I keep a VMWare snapshot of a clean Windows installation
+ for this, but that's probably overkill 8-) and test the latter on Unix
+ (a Debian VMWare box in my case).
+ o Dance the SourceForge release dance:
+ http://sourceforge.net/docman/display_doc.php?docid=6445&group_id=1#filereleasesteps
+ When it comes to the "what's new" and the ChangeLog, I cut'n'paste the
+ relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and
+ check the "Keep my preformatted text" checkbox.
+
+ The announce the release on spambayes-announce, spambayes and spambayes-dev
+ (why do things by halves?) and whatch the bug reports roll in. 8-)
From richiehindle at users.sourceforge.net Fri Sep 5 01:16:38 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Fri Sep 5 03:16:41 2003
Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv2200
Modified Files:
README-DEVEL.txt
Log Message:
Today's Top Tip: It helps if you hit the Save button before committing.
Index: README-DEVEL.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** README-DEVEL.txt 5 Sep 2003 07:06:50 -0000 1.2
--- README-DEVEL.txt 5 Sep 2003 07:16:36 -0000 1.3
***************
*** 333,340 ****
8-)
o Checkout the 'spambayes' module twice, once with Windows line endings
and once with Unix line endings (I use WinCVS for this, using "Admin /
Preferences / Globals / Checkout text files with the Unix LF".
! o Change spambayes/__init__.py to contain the new version number.
o In the Windows checkout, run "python setup.py sdist --formats zip"
o In the Unix checkout, run "python setup.py sdist --formats gztar"
--- 333,346 ----
8-)
+ o If any new file types have been added since last time (eg. 1.0a5 went
+ out without the Windows .rc and .h files) then add them to MANIFEST.in.
+ If there are any new scripts or packages, add them to setup.py. Test
+ these changes (by building source packages according to the instructions
+ below) then commit your edits.
o Checkout the 'spambayes' module twice, once with Windows line endings
and once with Unix line endings (I use WinCVS for this, using "Admin /
Preferences / Globals / Checkout text files with the Unix LF".
! o Change spambayes/__init__.py to contain the new version number but don't
! commit it yet, just in case something goes wrong.
o In the Windows checkout, run "python setup.py sdist --formats zip"
o In the Unix checkout, run "python setup.py sdist --formats gztar"
***************
*** 349,353 ****
relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and
check the "Keep my preformatted text" checkbox.
! The announce the release on spambayes-announce, spambayes and spambayes-dev
! (why do things by halves?) and whatch the bug reports roll in. 8-)
--- 355,362 ----
relevant pieces of WHAT_IS_NEW.txt and CHANGELOG.txt into the form, and
check the "Keep my preformatted text" checkbox.
+ o Now commit spambayes/__init__.py and tag the whole checkout - see the
+ existing tag names for the tag name format.
+ o Update the website News section.
! Then announce the release on the mailing lists and watch the bug reports
! roll in. 8-)
From mhammond at users.sourceforge.net Fri Sep 5 01:21:16 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 03:21:26 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py, 1.72,
1.73 tester.py, 1.17, 1.18
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv3001
Modified Files:
msgstore.py tester.py
Log Message:
Guard against invalid folder IDs better, and beef up the test suite to
check this and a few other things.
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.72
retrieving revision 1.73
diff -C2 -d -r1.72 -r1.73
*** msgstore.py 4 Sep 2003 12:14:12 -0000 1.72
--- msgstore.py 5 Sep 2003 07:21:14 -0000 1.73
***************
*** 57,64 ****
def __str__(self):
try:
! return "%s: %s" % (self.__class__.__name__,
! GetCOMExceptionString(self.mapi_exception))
except:
! print "Error __str__"
import traceback
traceback.print_exc()
--- 57,69 ----
def __str__(self):
try:
! if self.mapi_exception is not None:
! err_str = GetCOMExceptionString(self.mapi_exception)
! else:
! err_str = self.extra_msg or ''
! return "%s: %s" % (self.__class__.__name__, err_str)
! # Python silently consumes exceptions here, and uses
! #
except:
! print "FAILED to str() a MsgStore exception!"
import traceback
traceback.print_exc()
***************
*** 244,249 ****
assert type(item_id)==type(()), \
"Item IDs must be a tuple (not a %r)" % item_id
! store_id, entry_id = item_id
! return mapi.BinFromHex(store_id), mapi.BinFromHex(entry_id)
def _GetSubFolderIter(self, folder):
--- 249,257 ----
assert type(item_id)==type(()), \
"Item IDs must be a tuple (not a %r)" % item_id
! try:
! store_id, entry_id = item_id
! return mapi.BinFromHex(store_id), mapi.BinFromHex(entry_id)
! except ValueError:
! raise MsgStoreException(None, "The specified ID '%s' is invalid" % (item_id,))
def _GetSubFolderIter(self, folder):
***************
*** 264,268 ****
def GetFolderGenerator(self, folder_ids, include_sub):
for folder_id in folder_ids:
! folder_id = self.NormalizeID(folder_id)
try:
folder = self._OpenEntry(folder_id)
--- 272,280 ----
def GetFolderGenerator(self, folder_ids, include_sub):
for folder_id in folder_ids:
! try:
! folder_id = self.NormalizeID(folder_id)
! except MsgStoreException, details:
! print "NOTE: Skipping invalid folder", details
! continue
try:
folder = self._OpenEntry(folder_id)
Index: tester.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** tester.py 4 Sep 2003 12:14:12 -0000 1.17
--- tester.py 5 Sep 2003 07:21:14 -0000 1.18
***************
*** 12,15 ****
--- 12,16 ----
from win32com.client import constants
+ import sys
from time import sleep
import copy
***************
*** 18,21 ****
--- 19,24 ----
import threading
+ import msgstore
+
from win32com.mapi import mapi, mapiutil
import pythoncom
***************
*** 33,36 ****
--- 36,50 ----
raise TestFailure(msg)
+ def AssertRaises(exception, func, *args):
+ try:
+ func(*args)
+ raise TestFailed("Function '%s' should have raised '%r', but it worked!" % \
+ (func, exception))
+ except:
+ exc_type = sys.exc_info()[0]
+ if exc_type == exception or issubclass(exc_type, exception):
+ return
+ raise
+
filter_event = threading.Event()
***************
*** 352,366 ****
def TestHamFilter(driver):
! _DoTestHamFilter(driver, driver.folder_watch)
! # Try again, with the secondary folder if it exists (it is likely one
! # is the inbox and one isn't, so may be useful)
! if driver.folder_watch_2 is not None:
! _DoTestHamFilter(driver, driver.folder_watch_2)
! # Now test our incremental train logic
! _DoTestHamTrain(driver, driver.folder_watch, driver.folder_watch_2)
! else:
! print "Skipping testing secondary watch folder filtering - " \
! "only one watch folder is configured"
! print "Created a Ham message, and saw it remain in place."
def TestUnsureFilter(driver):
--- 366,381 ----
def TestHamFilter(driver):
! # Execute the 'ham' test in every folder we watch.
! mgr = driver.manager
! gen = mgr.message_store.GetFolderGenerator(
! mgr.config.filter.watch_folder_ids,
! mgr.config.filter.watch_include_sub)
! num = 0
! for f in gen:
! print "Running ham filter tests on folder '%s'" % f.GetFQName()
! f = f.GetOutlookItem()
! _DoTestHamFilter(driver, f)
! num += 1
! print "Created a Ham message, and saw it remain in place (in %d watch folders.)" % num
def TestUnsureFilter(driver):
***************
*** 442,446 ****
# Check messages we are unable to score.
# Must enable the filtering code for this test
- import msgstore
msgstore.test_suite_running = False
try:
--- 457,460 ----
***************
*** 482,485 ****
--- 496,524 ----
msgstore.test_suite_running = True
+ def run_invalid_id_tests(manager):
+ # Do some tests with invalid message and folder IDs.
+ print "Doing some 'invalid ID' tests - you should see a couple of warning, but no errors or tracebacks"
+ id_no_item = ('0000','0000') # this ID is 'valid' - but there will be no such item
+ id_invalid = ('xxxx','xxxx') # this ID is 'invalid' in that the hex-bin conversion fails
+ id_empty1 = ('','')
+ id_empty2 = ()
+ bad_ids = id_no_item, id_invalid, id_empty1, id_empty2
+ for id in bad_ids:
+ AssertRaises(msgstore.MsgStoreException, manager.message_store.GetMessage, id)
+ # Test 'GetFolderGenerator' works with invalid ids.
+ for id in bad_ids:
+ AssertRaises(msgstore.MsgStoreException, manager.message_store.GetFolder, id)
+ ids = manager.config.filter.watch_folder_ids[:]
+ ids.append(id)
+ found = 0
+ for f in manager.message_store.GetFolderGenerator(ids, False):
+ found += 1
+ if found > len(manager.config.filter.watch_folder_ids):
+ raise TestFailed("Seemed to find the extra folder")
+ names = manager.FormatFolderNames(ids, False)
+ if names.find("
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv10448
Modified Files:
applications.ht download.ht
Log Message:
More references to 1.0a4. I've left the Outlook plug-in source instructions to get
1.0a4 since 1.0a5 is missing files.
Index: applications.ht
===================================================================
RCS file: /cvsroot/spambayes/website/applications.ht,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** applications.ht 20 Aug 2003 04:26:33 -0000 1.20
--- applications.ht 5 Sep 2003 08:02:54 -0000 1.21
***************
*** 33,37 ****
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!).
--- 33,37 ----
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!).
***************
*** 45,49 ****
Availability
! Download the alpha4 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 45,49 ----
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
***************
*** 77,81 ****
Availability
! Download the alpha4 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 77,81 ----
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
***************
*** 93,97 ****
Availability
! Download the alpha4 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 93,97 ----
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
***************
*** 111,114 ****
Availability
! Download the alpha4 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 111,114 ----
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
Index: download.ht
===================================================================
RCS file: /cvsroot/spambayes/website/download.ht,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** download.ht 2 Sep 2003 03:41:55 -0000 1.16
--- download.ht 5 Sep 2003 08:02:54 -0000 1.17
***************
*** 4,9 ****
Source Releases
! The fourth pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a4 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
--- 4,9 ----
Source Releases
! The fifth pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a5 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
***************
*** 34,38 ****
see here for
more details. Note that due to capacity problems with Sourceforge,
! the public CVS servers often run up to 24 hours behind the real CVS
servers. This is something that SF are working on improving.
--- 34,38 ----
see here for
more details. Note that due to capacity problems with Sourceforge,
! the public CVS servers often run up to 48 hours behind the real CVS
servers. This is something that SF are working on improving.
From anadelonbrin at users.sourceforge.net Fri Sep 5 02:04:54 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 5 04:04:58 2003
Subject: [Spambayes-checkins] spambayes README-DEVEL.txt,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10746
Modified Files:
README-DEVEL.txt
Log Message:
Add references to the other pages that the version number is referenced on.
Index: README-DEVEL.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README-DEVEL.txt,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** README-DEVEL.txt 5 Sep 2003 07:16:36 -0000 1.3
--- README-DEVEL.txt 5 Sep 2003 08:04:51 -0000 1.4
***************
*** 357,361 ****
o Now commit spambayes/__init__.py and tag the whole checkout - see the
existing tag names for the tag name format.
! o Update the website News section.
Then announce the release on the mailing lists and watch the bug reports
--- 357,361 ----
o Now commit spambayes/__init__.py and tag the whole checkout - see the
existing tag names for the tag name format.
! o Update the website News, Download and Application sections.
Then announce the release on the mailing lists and watch the bug reports
From anadelonbrin at users.sourceforge.net Fri Sep 5 03:21:04 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 5 05:21:09 2003
Subject: [Spambayes-checkins] website faq.txt,1.39,1.40
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv22688
Modified Files:
faq.txt
Log Message:
Add a FAQ about the 'access denied' problem that a lot of pop3proxy
people seem to run into. We might want to special case this error and
tell them what to do instead, or maybe even just automatically try a
range of different ports.
Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.39
retrieving revision 1.40
diff -C2 -d -r1.39 -r1.40
*** faq.txt 27 Aug 2003 03:53:03 -0000 1.39
--- faq.txt 5 Sep 2003 09:21:02 -0000 1.40
***************
*** 933,936 ****
--- 933,947 ----
+ I've just installed SpamBayes, but when I run it I get an access denied error.
+ ------------------------------------------------------------------------------
+
+ If you use pop3proxy or imapfilter, haven't set anything up, and run it and
+ get an error that ends with ``socket.error: (10013, 'Permission denied')``,
+ then this probably means that port 8880, which SpamBayes is trying to use to
+ present the web interface, is already taken on your machine. Try using
+ ``pop3proxy.py -u 8881 -b`` (or ``imapfilter.py -u 8881 -b``), or another
+ port that you know is free and available on your machine.
+
+
Known Problems & Workarounds
============================
From mhammond at users.sourceforge.net Fri Sep 5 05:49:49 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 07:49:54 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/docs configuration.html,
1.8, 1.9
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/docs
In directory sc8-pr-cvs1:/tmp/cvs-serv14014/docs
Modified Files:
configuration.html
Log Message:
Clarify rules for "data_directory" based on bug report.
Index: configuration.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/configuration.html,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** configuration.html 26 Aug 2003 02:35:45 -0000 1.8
--- configuration.html 5 Sep 2003 11:49:47 -0000 1.9
***************
*** 186,190 ****
|
The directory where SpamBayes
! will save its database and primary configuration file. This can
only be set in the 'early' configuration files - see below.
|
--- 186,193 ----
|
The directory where SpamBayes
! will save its database and primary configuration file. This
! directory name should not be enclosed in quotes, but can include spaces
! and extended characters (in which case the default windows encoding is
! assumed). This can
only be set in the 'early' configuration files - see below.
|
From mhammond at users.sourceforge.net Fri Sep 5 05:50:59 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 07:51:03 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs dialog_map.py,
1.32, 1.33
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv14164/dialogs
Modified Files:
dialog_map.py
Log Message:
Claify wording when untrained, so it doesn't sound like 'unsure' is a
folder name.
Index: dialog_map.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/dialog_map.py,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** dialog_map.py 29 Aug 2003 06:23:50 -0000 1.32
--- dialog_map.py 5 Sep 2003 11:50:57 -0000 1.33
***************
*** 42,47 ****
else:
db_status = "Database has no training information. SpamBayes " \
! "will deliver all messages to your 'Unsure' folder, " \
! "ready for you to classify."
win32gui.SendMessage(self.GetControl(), win32con.WM_SETTEXT,
0, db_status)
--- 42,47 ----
else:
db_status = "Database has no training information. SpamBayes " \
! "will classify all messages as 'unsure', " \
! "ready for you to train."
win32gui.SendMessage(self.GetControl(), win32con.WM_SETTEXT,
0, db_status)
From mhammond at users.sourceforge.net Fri Sep 5 05:54:21 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 5 07:54:26 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py, 1.84,
1.85 msgstore.py, 1.73, 1.74
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv14453
Modified Files:
manager.py msgstore.py
Log Message:
Exception handling functions also decode Outlook exceptions (which are
'extended' exceptions, with the underlying MAPI code buried).
When we fail to add the 'Spam' field to a read-only store (eg, hotmail),
complain less loudly.
Index: manager.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v
retrieving revision 1.84
retrieving revision 1.85
diff -C2 -d -r1.84 -r1.85
*** manager.py 4 Sep 2003 12:14:12 -0000 1.84
--- manager.py 5 Sep 2003 11:54:19 -0000 1.85
***************
*** 551,556 ****
folder = msgstore_folder.GetOutlookItem()
self.LogDebug(2, "Checking folder '%s' for field '%s'" \
! % (folder.Name.encode("mbcs", "replace"), self.config.general.field_score_name))
items = folder.Items
item = items.GetFirst()
--- 551,557 ----
folder = msgstore_folder.GetOutlookItem()
+ folder_name = msgstore_folder.GetFQName()
self.LogDebug(2, "Checking folder '%s' for field '%s'" \
! % (folder_name, self.config.general.field_score_name))
items = folder.Items
item = items.GetFirst()
***************
*** 581,588 ****
self.LogDebug(2, "Created the UserProperty!")
except pythoncom.com_error, details:
! print "Warning: failed to create the Outlook " \
! "user-property in folder '%s'" \
! % (folder.Name.encode("mbcs", "replace"),)
! print "", details
if include_sub:
# Recurse down the folder list.
--- 582,593 ----
self.LogDebug(2, "Created the UserProperty!")
except pythoncom.com_error, details:
! if msgstore.IsReadOnlyCOMException(details):
! self.LogDebug(1, "The folder '%s' is read-only - user property can't be added" % \
! (folder_name,))
! else:
! print "Warning: failed to create the Outlook " \
! "user-property in folder '%s'" \
! % (folder_name,)
! print "", details
if include_sub:
# Recurse down the folder list.
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.73
retrieving revision 1.74
diff -C2 -d -r1.73 -r1.74
*** msgstore.py 5 Sep 2003 07:21:14 -0000 1.73
--- msgstore.py 5 Sep 2003 11:54:19 -0000 1.74
***************
*** 15,18 ****
--- 15,19 ----
from win32com.mapi.mapitags import *
import pythoncom
+ import winerror
# Additional MAPI constants we dont have in Python
***************
*** 83,86 ****
--- 84,91 ----
pass
+ # The object has changed since it was opened.
+ class ObjectChangedException(MsgStoreException):
+ pass
+
# Utility functions for exceptions. Convert a COM exception to the best
# manager exception.
***************
*** 90,98 ****
if IsReadOnlyCOMException(com_exc):
return ReadOnlyException(com_exc)
return MsgStoreException(com_exc)
# Build a reasonable string from a COM exception tuple
def GetCOMExceptionString(exc_val):
! hr, msg, exc, arg_err = exc_val
err_string = mapiutil.GetScodeString(hr)
return "Exception 0x%x (%s): %s" % (hr, err_string, msg)
--- 95,114 ----
if IsReadOnlyCOMException(com_exc):
return ReadOnlyException(com_exc)
+ scode = NormalizeCOMException(com_exc)[0]
+ # And simple scode based ones.
+ if scode == mapi.MAPI_E_OBJECT_CHANGED:
+ return ObjectChangedException(com_exc)
return MsgStoreException(com_exc)
+ def NormalizeCOMException(exc_val):
+ hr, msg, exc, arg_err = exc_val
+ if hr == winerror.DISP_E_EXCEPTION and exc:
+ # 'client' exception - unpack 'exception object'
+ wcode, source, msg, help1, help2, hr = exc
+ return hr, msg, exc, arg_err
+
# Build a reasonable string from a COM exception tuple
def GetCOMExceptionString(exc_val):
! hr, msg, exc, arg_err = NormalizeCOMException(exc_val)
err_string = mapiutil.GetScodeString(hr)
return "Exception 0x%x (%s): %s" % (hr, err_string, msg)
***************
*** 100,104 ****
# Does this exception probably mean "object not found"?
def IsNotFoundCOMException(exc_val):
! hr, msg, exc, arg_err = exc_val
return hr in [mapi.MAPI_E_OBJECT_DELETED, mapi.MAPI_E_NOT_FOUND]
--- 116,120 ----
# Does this exception probably mean "object not found"?
def IsNotFoundCOMException(exc_val):
! hr, msg, exc, arg_err = NormalizeCOMException(exc_val)
return hr in [mapi.MAPI_E_OBJECT_DELETED, mapi.MAPI_E_NOT_FOUND]
***************
*** 106,110 ****
# in, or 'cos the server is down"?
def IsNotAvailableCOMException(exc_val):
! hr, msg, exc, arg_err = exc_val
return hr == mapi.MAPI_E_FAILONEPROVIDER
--- 122,126 ----
# in, or 'cos the server is down"?
def IsNotAvailableCOMException(exc_val):
! hr, msg, exc, arg_err = NormalizeCOMException(exc_val)
return hr == mapi.MAPI_E_FAILONEPROVIDER
***************
*** 113,118 ****
# and also for hotmail messages (0x8004dff7)
known_failure_codes = -2146644781, -2147164169
return exc_val[0] in known_failure_codes
-
def ReportMAPIError(manager, what, exc_val):
--- 129,134 ----
# and also for hotmail messages (0x8004dff7)
known_failure_codes = -2146644781, -2147164169
+ exc_val = NormalizeCOMException(exc_val)
return exc_val[0] in known_failure_codes
def ReportMAPIError(manager, what, exc_val):
From montanaro at users.sourceforge.net Fri Sep 5 09:18:06 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 5 11:18:11 2003
Subject: [Spambayes-checkins] spambayes/spambayes hammie.py,1.8,1.9
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv23782
Modified Files:
hammie.py
Log Message:
hammie_train_on_filter -> train_on_filter - option change was missed
Index: hammie.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** hammie.py 5 Sep 2003 01:15:28 -0000 1.8
--- hammie.py 5 Sep 2003 15:18:03 -0000 1.9
***************
*** 99,103 ****
debug = options["Hammie", "debug_header"]
if train == None:
! train = options["Hammie", "hammie_train_on_filter"]
msg = mboxutils.get_message(msg)
--- 99,103 ----
debug = options["Hammie", "debug_header"]
if train == None:
! train = options["Hammie", "train_on_filter"]
msg = mboxutils.get_message(msg)
From montanaro at users.sourceforge.net Fri Sep 5 09:25:38 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 5 11:25:42 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_chkopts.py,NONE,1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv25157
Added Files:
sb_chkopts.py
Log Message:
trivial options checker
--- NEW FILE: sb_chkopts.py ---
#!/usr/bin/env python
"""
Trivial script to check that the user's options file doesn't contain any
old-style option names.
"""
from spambayes import Options
From anadelonbrin at users.sourceforge.net Fri Sep 5 21:12:49 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 5 23:12:54 2003
Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27451
Modified Files:
MANIFEST.in
Log Message:
Fix [ 800555 ] 1.0a5 release missing key outlook files.
Index: MANIFEST.in
===================================================================
RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** MANIFEST.in 7 Jul 2003 00:09:27 -0000 1.5
--- MANIFEST.in 6 Sep 2003 03:12:46 -0000 1.6
***************
*** 3,7 ****
recursive-include pspam *.py *.txt *.ini *.sh
recursive-include contrib *.py *.sh *.txt *.el *rc
! recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp
recursive-include utilities *.py *.txt
recursive-include testtools *.py *.txt
--- 3,7 ----
recursive-include pspam *.py *.txt *.ini *.sh
recursive-include contrib *.py *.sh *.txt *.el *rc
! recursive-include Outlook2000 *.py *.txt *.ini *.html *.bmp *.rc *.h
recursive-include utilities *.py *.txt
recursive-include testtools *.py *.txt
From anadelonbrin at users.sourceforge.net Fri Sep 5 22:03:37 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 6 00:03:40 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_pop3dnd.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv1864/scripts
Modified Files:
sb_pop3dnd.py
Log Message:
Opps. I should have done this before. Add a missing file that's necessary to try out the
sb_pop3dnd.py script, and the version and options necessary. Also update to reflect
the pop3proxy.py renaming.
Index: sb_pop3dnd.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_pop3dnd.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_pop3dnd.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_pop3dnd.py 6 Sep 2003 04:03:34 -0000 1.2
***************
*** 3,8 ****
from __future__ import generators
! """
! Overkill (someone *please* come up with something to call this script!)
This application is a twisted cross between a POP3 proxy and an IMAP
--- 3,7 ----
from __future__ import generators
! """POP3DND - provides drag'n'drop training ability for POP3 clients.
This application is a twisted cross between a POP3 proxy and an IMAP
***************
*** 133,137 ****
from spambayes.ServerUI import ServerUserInterface
from spambayes.UserInterface import UserInterfaceServer
! from pop3proxy import POP3ProxyBase, State, _addressPortStr, _recreateState
def ensureDir(dirname):
--- 132,136 ----
from spambayes.ServerUI import ServerUserInterface
from spambayes.UserInterface import UserInterfaceServer
! from sb_server import POP3ProxyBase, State, _addressPortStr, _recreateState
def ensureDir(dirname):
***************
*** 928,932 ****
# Let the user know what they are using...
print get_version_string("IMAP Server")
! print "and engine %s.\n" % (get_version_string(),)
# setup everything
--- 927,934 ----
# Let the user know what they are using...
print get_version_string("IMAP Server")
! print get_version_string("POP3 Proxy")
! print "and engine %s," % (get_version_string(),)
! from twisted.copyright import version as twisted_version
! print "with twisted version %s.\n" % (twisted_version,)
# setup everything
From anadelonbrin at users.sourceforge.net Fri Sep 5 22:03:37 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 6 00:03:43 2003
Subject: [Spambayes-checkins] spambayes/spambayes ServerUI.py, NONE,
1.1 Options.py, 1.70, 1.71 Version.py, 1.17, 1.18
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1864/spambayes
Modified Files:
Options.py Version.py
Added Files:
ServerUI.py
Log Message:
Opps. I should have done this before. Add a missing file that's necessary to try out the
sb_pop3dnd.py script, and the version and options necessary. Also update to reflect
the pop3proxy.py renaming.
--- NEW FILE: ServerUI.py ---
"""IMAP Server Web Interface
Classes:
ServerInterface - Interface class for imapserver
Abstract:
This module implements a browser based Spambayes user interface for the
IMAP server. Users may use it to interface with the server.
The following functions are currently included:
[From the base class UserInterface]
onClassify - classify a given message
onWordquery - query a word from the database
onTrain - train a message or mbox
onSave - save the database and possibly shutdown
[Here]
onHome - a home page with various options
To do:
o Suggestions?
"""
# This module is part of the spambayes project, which is Copyright 2002-3
# The Python Software Foundation and is covered by the Python Software
# Foundation license.
__author__ = "Tony Meyer "
__credits__ = "All the Spambayes folk."
try:
True, False
except NameError:
# Maintain compatibility with Python 2.2
True, False = 1, 0
from spambayes import UserInterface
from spambayes.Options import options
# These are the options that will be offered on the configuration page.
# If the option is None, then the entry is a header and the following
# options will appear in a new box on the configuration page.
# These are also used to generate http request parameters and template
# fields/variables.
parm_ini_map = (
('POP3 Proxy Options', None),
('pop3proxy', 'remote_servers'),
('pop3proxy', 'listen_ports'),
('IMAP Server Options', None),
('imapserver', 'username'),
('imapserver', 'password'),
('imapserver', 'port'),
('Storage Options', None),
('Storage', 'persistent_storage_file'),
('Storage', 'messageinfo_storage_file'),
('Statistics Options', None),
('Categorization', 'ham_cutoff'),
('Categorization', 'spam_cutoff'),
('Classifier', 'experimental_ham_spam_imbalance_adjustment'),
)
class ServerUserInterface(UserInterface.UserInterface):
"""Serves the HTML user interface for the server."""
def __init__(self, state, state_recreator):
UserInterface.UserInterface.__init__(self, state.bayes,
parm_ini_map)
self.state = state
self.state_recreator = state_recreator
def onHome(self):
"""Serve up the homepage."""
stateDict = self.state.__dict__.copy()
stateDict.update(self.state.bayes.__dict__)
statusTable = self.html.statusTable.clone()
if not self.state.servers:
statusTable.proxyDetails = "No POP3 proxies running."
content = (self._buildBox('Status and Configuration',
'status.gif', statusTable % stateDict)+
self._buildTrainBox() +
self._buildClassifyBox() +
self._buildBox('Word query', 'query.gif',
self.html.wordQuery)
)
self._writePreamble("Home")
self.write(content)
self._writePostamble()
def reReadOptions(self):
"""Called by the config page when the user saves some new options,
or restores the defaults."""
# Reload the options.
self.state.bayes.store()
import Options
reload(Options)
global options
from Options import options
# Recreate the state.
self.state = self.state_recreator()
def verifyInput(self, parms):
'''Check that the given input is valid.'''
# Most of the work here is done by the parent class, but
# we have a few extra checks
errmsg = UserInterface.UserInterface.verifyInput(self, parms)
# check for equal number of pop3servers and ports
slist = list(parms['pop3proxy_remote_servers'])
plist = list(parms['pop3proxy_listen_ports'])
if len(slist) != len(plist):
errmsg += 'The number of POP3 proxy ports specified ' + \
'must match the number of servers specified\n'
# check for duplicate ports
plist.sort()
for p in range(len(plist)-1):
try:
if plist[p] == plist[p+1]:
errmsg += 'All POP3 port numbers must be unique'
break
except IndexError:
pass
return errmsg
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.70
retrieving revision 1.71
diff -C2 -d -r1.70 -r1.71
*** Options.py 5 Sep 2003 01:15:28 -0000 1.70
--- Options.py 6 Sep 2003 04:03:35 -0000 1.71
***************
*** 892,895 ****
--- 892,921 ----
),
+ "imapserver" : (
+ ("username", "Username", "",
+ """The username to use when logging into the SpamBayes IMAP server.""",
+ IMAP_ASTRING, DO_NOT_RESTORE),
+
+ ("password", "Password", "",
+ """The password to use when logging into the SpamBayes IMAP server.""",
+ IMAP_ASTRING, DO_NOT_RESTORE),
+
+ ("port", "IMAP Listen Port", 143,
+ """The port to serve the SpamBayes IMAP server on.""",
+ PORT, RESTORE),
+
+ ("spam_directory", "Spam directory", "imapserver-spam",
+ """The directory to store spam messages in.""",
+ PATH, DO_NOT_RESTORE),
+
+ ("ham_directory", "Ham directory", "imapserver-ham",
+ """The directory to store ham messages in.""",
+ PATH, DO_NOT_RESTORE),
+
+ ("unsure_directory", "Unsure directory", "imapserver-unsure",
+ """The directory to store unsure messages in.""",
+ PATH, DO_NOT_RESTORE),
+ ),
+
"globals" : (
("verbose", "Verbose", False,
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** Version.py 2 Sep 2003 01:31:04 -0000 1.17
--- Version.py 6 Sep 2003 04:03:35 -0000 1.18
***************
*** 73,76 ****
--- 73,85 ----
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
},
+ "IMAP Server" : {
+ "Version": 0.01,
+ "Description": "SpamBayes IMAP Server Alpha1",
+ "Date": "September 2003",
+ "InterfaceVersion": 0.02,
+ "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2",
+ "Full Description": """%(Description)s, version %(Version)s (%(Date)s),
+ using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
+ },
},
}
From mhammond at users.sourceforge.net Sat Sep 6 21:53:16 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sat Sep 6 23:53:19 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
async_processor.py, 1.7, 1.8
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv18595
Modified Files:
async_processor.py
Log Message:
Handle ranges that don't quite have enough ticks or slightly too many.
Don't use the underlying "tick" methods - just set the pos directly, as
we know it.
This fixes the problem that seemed to stop the progress bar from ever
getting over 80% or so.
Index: async_processor.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/async_processor.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** async_processor.py 1 Sep 2003 05:34:19 -0000 1.7
--- async_processor.py 7 Sep 2003 03:53:13 -0000 1.8
***************
*** 13,16 ****
--- 13,17 ----
True, False = 1, 0
+ verbose = 0
IDC_START = 1100
***************
*** 50,54 ****
win32api.PostMessage(self.hprogress, commctrl.PBM_SETSTEP, 1, 0)
win32api.PostMessage(self.hprogress, commctrl.PBM_SETPOS, 0, 0)
- self.current_control_tick = 0
self.current_stage += 1
--- 51,54 ----
***************
*** 59,68 ****
def set_max_ticks(self, m):
self._next_stage()
- self.current_stage_tick = 0
self.current_stage_max = m
def tick(self):
! self.current_stage_tick += 1
# Calc how far through this stage.
this_prop = float(self.current_stage_tick) / self.current_stage_max
--- 59,73 ----
def set_max_ticks(self, m):
+ # skip to the stage.
self._next_stage()
self.current_stage_max = m
+ self.current_stage_tick = -1 # ready to go to zero!
+ # if earlier stages stopped early, skip ahead.
+ self.tick()
def tick(self):
! if self.current_stage_tick < self.current_stage_max:
! # Don't let us go beyond our stage max
! self.current_stage_tick += 1
# Calc how far through this stage.
this_prop = float(self.current_stage_tick) / self.current_stage_max
***************
*** 75,83 ****
# user knows the process has actually started.)
control_tick = max(1,int(total_prop * self.total_control_ticks))
! #print "Tick", self.current_stage_tick, "is", this_prop, "through the stage,", total_prop, "through the total - ctrl tick is", control_tick
! while self.current_control_tick < control_tick:
! self.current_control_tick += 1
! #print "ticking control", self.current_control_tick
! win32api.PostMessage(self.hprogress, commctrl.PBM_STEPIT, 0, 0)
def _get_stage_text(self, text):
--- 80,86 ----
# user knows the process has actually started.)
control_tick = max(1,int(total_prop * self.total_control_ticks))
! if verbose:
! print "Tick", self.current_stage_tick, "is", this_prop, "through the stage,", total_prop, "through the total - ctrl tick is", control_tick
! win32api.PostMessage(self.hprogress, commctrl.PBM_SETPOS, control_tick)
def _get_stage_text(self, text):
***************
*** 237,240 ****
--- 240,244 ----
if __name__=='__main__':
+ verbose = 1
# Test my "multi-stage" code
class HackProgress(_Progress):
***************
*** 243,250 ****
self.dlg = None
self.stopping = False
! self.total_control_ticks = 100
self.current_stage = 0
self.set_stages( (("", 1.0),) )
p = HackProgress()
p.set_max_ticks(10)
--- 247,255 ----
self.dlg = None
self.stopping = False
! self.total_control_ticks = 40
self.current_stage = 0
self.set_stages( (("", 1.0),) )
+ print "Single stage test"
p = HackProgress()
p.set_max_ticks(10)
***************
*** 252,255 ****
--- 257,261 ----
p.tick()
+ print "First stage test"
p = HackProgress()
stages = ("Stage 1", 0.2), ("Stage 2", 0.8)
***************
*** 260,265 ****
p.tick()
# Do stage 2
! p.set_max_ticks(1000)
! for i in range(1000):
p.tick()
print "Done!"
--- 266,292 ----
p.tick()
# Do stage 2
! p.set_max_ticks(20)
! for i in range(20):
! p.tick()
! print "Second stage test"
! p = HackProgress()
! stages = ("Stage 1", 0.9), ("Stage 2", 0.1)
! p.set_stages(stages)
! p.set_max_ticks(10)
! for i in range(7): # do a few less just to check
! p.tick()
! p.set_max_ticks(2)
! for i in range(2):
! p.tick()
! print "Third stage test"
! p = HackProgress()
! stages = ("Stage 1", 0.9), ("Stage 2", 0.1)
! p.set_stages(stages)
! p.set_max_ticks(300)
! for i in range(313): # do a few more just to check
! p.tick()
! p.set_max_ticks(2)
! for i in range(2):
p.tick()
+
print "Done!"
From mhammond at users.sourceforge.net Sun Sep 7 17:38:03 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 7 19:38:09 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/docs
troubleshooting.html, 1.14, 1.15
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/docs
In directory sc8-pr-cvs1:/tmp/cvs-serv14905/docs
Modified Files:
troubleshooting.html
Log Message:
Instructions how to locate the toolbar to delete assumed that the
'toolbars' tab would be the current tab.
Index: troubleshooting.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/troubleshooting.html,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** troubleshooting.html 29 Aug 2003 06:24:48 -0000 1.14
--- troubleshooting.html 7 Sep 2003 23:38:01 -0000 1.15
***************
*** 46,50 ****
- Right-click on any Outlook toolbar, and select Customize.
! - In the dialog that appears, locate SpamBayes in the list of
toolbars, and select it.
- Click on the Delete button. Outlook will ask for
--- 46,52 ----
- Right-click on any Outlook toolbar, and select Customize.
! - In the dialog that appears, ensure the Toolbars tab is selected, locate
! SpamBayes in the list of
toolbars, and select it.
- Click on the Delete button. Outlook will ask for
From mhammond at users.sourceforge.net Sun Sep 7 17:42:22 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 7 19:42:25 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py, 1.85,
1.86 addin.py, 1.107, 1.108
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv15530
Modified Files:
manager.py addin.py
Log Message:
When processing a hotmail inbox, we could see "object changed" exceptions
reported by MAPI. Simply starting again on this exception seems to work.
Index: manager.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v
retrieving revision 1.85
retrieving revision 1.86
diff -C2 -d -r1.85 -r1.86
*** manager.py 5 Sep 2003 11:54:19 -0000 1.85
--- manager.py 7 Sep 2003 23:42:19 -0000 1.86
***************
*** 457,464 ****
return
print "ERROR:", repr(msg)
- traceback.print_exc()
if key in self.reported_error_map:
print "(this error has already been reported - not displaying it again)"
else:
self.reported_error_map[key] = True
ReportError(msg, title)
--- 457,464 ----
return
print "ERROR:", repr(msg)
if key in self.reported_error_map:
print "(this error has already been reported - not displaying it again)"
else:
+ traceback.print_exc()
self.reported_error_map[key] = True
ReportError(msg, title)
***************
*** 786,789 ****
--- 786,791 ----
"must re-train the system via the SpamBayes manager."
self.ReportErrorOnce(msg)
+ # and disable the addin, as we are hosed!
+ self.config.filter.enabled = False
raise
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.107
retrieving revision 1.108
diff -C2 -d -r1.107 -r1.108
*** addin.py 4 Sep 2003 12:14:11 -0000 1.107
--- addin.py 7 Sep 2003 23:42:19 -0000 1.108
***************
*** 373,383 ****
if not self.use_timer or not item.UnRead:
ms = self.manager.message_store
! try:
! msgstore_message = ms.GetMessage(item)
! ProcessMessage(msgstore_message, self.manager)
! except ms.MsgStoreException, details:
! print "Unexpected error fetching message"
! traceback.print_exc()
! print details
else:
self._StartTimer()
--- 373,397 ----
if not self.use_timer or not item.UnRead:
ms = self.manager.message_store
! # The object can sometimes change underneath us (most noticably
! # with hotmail, which presumably has greater 'synchronization'
! # issues than tightly bound stores. We accept this error exactly
! # 2 times (once as we have seen it, and twice as it can't hurt ;)
! for i in range(3):
! try:
! msgstore_message = ms.GetMessage(item)
! ProcessMessage(msgstore_message, self.manager)
! break
! except ms.ObjectChangedException:
! # try again.
! self.manager.LogDebug(1, "Got object changed error - trying again...")
! continue
! except ms.MsgStoreException, details:
! print "Unexpected error fetching message"
! traceback.print_exc()
! print details
! break
! else:
! print "WARNING: Could not filter a message, as we got an " \
! "'object changed' exception 3 times!"
else:
self._StartTimer()
From mhammond at users.sourceforge.net Sun Sep 7 17:43:23 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 7 19:43:26 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.74,1.75
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv15852
Modified Files:
msgstore.py
Log Message:
Expose 'ObjectChanged' exception in the class as well as the module.
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.74
retrieving revision 1.75
diff -C2 -d -r1.74 -r1.75
*** msgstore.py 5 Sep 2003 11:54:19 -0000 1.74
--- msgstore.py 7 Sep 2003 23:43:21 -0000 1.75
***************
*** 154,157 ****
--- 154,158 ----
NotFoundException = NotFoundException
ReadOnlyException = ReadOnlyException
+ ObjectChangedException = ObjectChangedException
def __init__(self, outlook = None):
From mhammond at users.sourceforge.net Sun Sep 7 18:14:16 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 7 20:14:20 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.108,1.109
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv20201
Modified Files:
addin.py
Log Message:
Fix [ 798362 ] Toolbar becomes unresponsive and must be recreated
If we fail to add an item to a popup, assume it is dead, so delete and
re-create it.
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.108
retrieving revision 1.109
diff -C2 -d -r1.108 -r1.109
*** addin.py 7 Sep 2003 23:42:19 -0000 1.108
--- addin.py 8 Sep 2003 00:14:14 -0000 1.109
***************
*** 760,772 ****
# The main tool-bar dropdown with all our entries.
# Add a pop-up menu to the toolbar
! popup = self._AddControl(
! None,
! constants.msoControlPopup,
! None, None,
! Caption="SpamBayes",
! TooltipText = "SpamBayes anti-spam filters and functions",
! Enabled = True,
! Tag = "SpamBayesCommand.Popup")
! if popup is not None: # We may not be able to find/create our button
# Convert from "CommandBarItem" to derived
# "CommandBarPopup" Not sure if we should be able to work
--- 760,778 ----
# The main tool-bar dropdown with all our entries.
# Add a pop-up menu to the toolbar
! # but loop around twice - first time we may find a non-functioning button
! popup = None
! for attempt in range(2):
! popup = self._AddControl(
! None,
! constants.msoControlPopup,
! None, None,
! Caption="SpamBayes",
! TooltipText = "SpamBayes anti-spam filters and functions",
! Enabled = True,
! Tag = "SpamBayesCommand.Popup")
! if popup is None:
! # If the strategy below works for child buttons, we should
! # consider trying to re-create the top-level toolbar too.
! break
# Convert from "CommandBarItem" to derived
# "CommandBarPopup" Not sure if we should be able to work
***************
*** 776,780 ****
popup = CastTo(popup, "CommandBarPopup")
# And add our children.
! self._AddControl(popup,
constants.msoControlButton,
ButtonEvent, (manager.ShowManager,),
--- 782,786 ----
popup = CastTo(popup, "CommandBarPopup")
# And add our children.
! child = self._AddControl(popup,
constants.msoControlButton,
ButtonEvent, (manager.ShowManager,),
***************
*** 784,787 ****
--- 790,812 ----
Visible=True,
Tag = "SpamBayesCommand.Manager")
+ # Only necessary to check the first child - if the first works,
+ # the others will too
+ if child is None:
+ # Try and delete the popup, the bounce around the loop again,
+ # which will re-create it.
+ try:
+ item = self.CommandBars.FindControl(
+ Type = constants.msoControlPopup,
+ Tag = "SpamBayesCommand.Popup")
+ if item is None:
+ print "ERROR: Could't re-find control to delete"
+ break
+ item.Delete(False)
+ print "Deleted the dead popup control - re-creating"
+ except pythoncom.com_error, e:
+ print "ERROR: Failed to delete our dead toolbar control"
+ break
+ # ok - toolbar deleted - just run around the loop again
+ continue
self._AddControl(popup,
constants.msoControlButton,
***************
*** 909,913 ****
if toolbar.Name == "SpamBayes":
self.toolbar = toolbar
- print "Found SB toolbar - visible state is", toolbar.Visible
break
else:
--- 934,937 ----
***************
*** 935,939 ****
# eg, bug [ 755738 ] Latest CVS outllok doesn't work
print "FAILED to add the toolbar item '%s' - %s" % (tag,e)
- traceback.print_exc()
return
if image_fname:
--- 959,962 ----
From mhammond at users.sourceforge.net Sun Sep 7 20:35:48 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 7 22:35:52 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 tester.py,1.18,1.19
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv9209
Modified Files:
tester.py
Log Message:
Run every test over every "watch" folder - this is useful for testing
hotmail/imap etc. IMAP still fails the test suite, but for reasons
related to the test suite itself.
Index: tester.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** tester.py 5 Sep 2003 07:21:14 -0000 1.18
--- tester.py 8 Sep 2003 02:35:46 -0000 1.19
***************
*** 140,158 ****
folder = mgr.message_store.GetFolder(mgr.config.filter.unsure_folder_id)
self.folder_unsure = folder.GetOutlookItem()
! # The "watch" folder is a folder we can stick stuff into to have them
! # filtered - just use the first one nominated.
! self.folder_watch = self.folder_watch_2 = None
gen = mgr.message_store.GetFolderGenerator(
mgr.config.filter.watch_folder_ids,
mgr.config.filter.watch_include_sub)
! try:
! self.folder_watch = gen.next().GetOutlookItem()
! self.folder_watch_2 = gen.next().GetOutlookItem()
! except StopIteration:
! pass
! if self.folder_watch is None:
! raise RuntimeError, "Can't test without at least one folder to watch"
! # And the drafts folder where new messages are created.
! self.folder_drafts = mgr.outlook.Session.GetDefaultFolder(constants.olFolderDrafts)
def FindTestMessage(self, folder):
--- 140,153 ----
folder = mgr.message_store.GetFolder(mgr.config.filter.unsure_folder_id)
self.folder_unsure = folder.GetOutlookItem()
! # And the drafts folder where new messages are created.
! self.folder_drafts = mgr.outlook.Session.GetDefaultFolder(constants.olFolderDrafts)
!
! def GetWatchFolderGenerator(self):
! mgr = self.manager
gen = mgr.message_store.GetFolderGenerator(
mgr.config.filter.watch_folder_ids,
mgr.config.filter.watch_include_sub)
! for f in gen:
! yield f, f.GetOutlookItem()
def FindTestMessage(self, folder):
***************
*** 161,182 ****
return items.Find("[Subject] = '%s'" % (subject,))
def _CleanTestMessageFromFolder(self, folder):
subject = TEST_SUBJECT
num = 0
! while True:
msg = self.FindTestMessage(folder)
if msg is None:
break
msg.Delete()
! num += 1
if num:
print "Cleaned %d test messages from folder '%s'" % (num, folder.Name)
def CleanAllTestMessages(self):
- subject = TEST_SUBJECT
self._CleanTestMessageFromFolder(self.folder_spam)
self._CleanTestMessageFromFolder(self.folder_unsure)
- self._CleanTestMessageFromFolder(self.folder_watch)
self._CleanTestMessageFromFolder(self.folder_drafts)
def CreateTestMessageInFolder(self, spam_status, folder):
--- 156,192 ----
return items.Find("[Subject] = '%s'" % (subject,))
+ def CheckMessageFilteredFrom(self, folder):
+ # For hotmail accounts, the message may take a little time to actually
+ # be removed from the original folder (ie, it appears in the "dest"
+ # folder before it vanished.
+ for i in range(5):
+ if self.FindTestMessage(folder) is None:
+ break
+ sleep(.5)
+ else:
+ TestFailed("The test message remained in folder %r" % folder.Name)
+
def _CleanTestMessageFromFolder(self, folder):
subject = TEST_SUBJECT
num = 0
! # imap/hotmail etc only soft delete, and I see no way to differentiate
! # force the user to purge them manually
! for i in range(50):
msg = self.FindTestMessage(folder)
if msg is None:
break
msg.Delete()
! else:
! raise TestFailed("Old test messages appear to still exist. These may" \
! "be 'soft-deleted' - you will need to purge them manually")
if num:
print "Cleaned %d test messages from folder '%s'" % (num, folder.Name)
def CleanAllTestMessages(self):
self._CleanTestMessageFromFolder(self.folder_spam)
self._CleanTestMessageFromFolder(self.folder_unsure)
self._CleanTestMessageFromFolder(self.folder_drafts)
+ for msf, of in self.GetWatchFolderGenerator():
+ self._CleanTestMessageFromFolder(of)
def CreateTestMessageInFolder(self, spam_status, folder):
***************
*** 223,327 ****
nham = bayes.nham
original_bayes = copy.copy(driver.manager.classifier_data.bayes)
! # Create a spam message in the Inbox - it should get immediately filtered
! msg, words = driver.CreateTestMessageInFolder(SPAM, driver.folder_watch)
! # sleep to ensure filtering.
! WaitForFilters()
! # It should no longer be in the Inbox.
! if driver.FindTestMessage(driver.folder_watch) is not None:
! TestFailed("The test message appeared to not be filtered")
! # It should be in the "sure is spam" folder.
! spam_msg = driver.FindTestMessage(driver.folder_spam)
! if spam_msg is None:
! TestFailed("The test message vanished from the Inbox, but didn't appear in Spam")
! # Check that none of the above caused training.
! if nspam != bayes.nspam:
! TestFailed("Something caused a new spam message to appear")
! if nham != bayes.nham:
! TestFailed("Something caused a new ham message to appear")
! check_words(words, bayes, 0, 0)
!
! # Now move the message back to the inbox - it should get trained.
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! import train
! if train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as ham yet")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as spam yet")
! spam_msg.Move(driver.folder_watch)
! WaitForFilters()
! spam_msg = driver.FindTestMessage(driver.folder_watch)
! if spam_msg is None:
! TestFailed("The message appears to have been filtered out of the watch folder")
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! need_untrain = True
! try:
! if nspam != bayes.nspam:
! TestFailed("There were not the same number of spam messages after a re-train")
! if nham+1 != bayes.nham:
! TestFailed("There was not one more ham messages after a re-train")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as spam yet")
! if not train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been trained as ham now")
! # word infos should have one extra ham
! check_words(words, bayes, 0, 1)
! # Now move it back to the Spam folder.
! # This should see the message un-trained as ham, and re-trained as Spam
! spam_msg.Move(driver.folder_spam)
WaitForFilters()
spam_msg = driver.FindTestMessage(driver.folder_spam)
if spam_msg is None:
! TestFailed("Could not find the message in the Spam folder")
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! if nspam +1 != bayes.nspam:
! TestFailed("There should be one more spam now")
if nham != bayes.nham:
! TestFailed("There should be the same number of hams again")
! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been trained as spam by now")
if train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been un-trained as ham")
! # word infos should have one extra spam, no extra ham
! check_words(words, bayes, 1, 0)
! # Move the message to another folder, and make sure we still
! # identify it correctly as having been trained.
! # Move to the "unsure" folder, just cos we know about it, and
! # we know that no special watching of this folder exists.
! spam_msg.Move(driver.folder_unsure)
! spam_msg = driver.FindTestMessage(driver.folder_unsure)
if spam_msg is None:
! TestFailed("Could not find the message in the Unsure folder")
store_msg = driver.manager.message_store.GetMessage(spam_msg)
! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("Message was not identified as Spam after moving")
!
! # word infos still be 'spam'
! check_words(words, bayes, 1, 0)
!
! # Now undo the damage we did.
! was_spam = train.untrain_message(store_msg, driver.manager.classifier_data)
! if not was_spam:
! TestFailed("Untraining this message did not indicate it was spam")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data) or \
! train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("Untraining this message kept it has ham/spam")
! need_untrain = False
! finally:
! if need_untrain:
! train.untrain_message(store_msg, driver.manager.classifier_data)
!
! # Check all the counts are back where we started.
! if nspam != bayes.nspam:
! TestFailed("Spam count didn't get back to the same")
! if nham != bayes.nham:
! TestFailed("Ham count didn't get back to the same")
! check_words(words, bayes, 0, 0)
!
! if bayes.wordinfo != original_bayes.wordinfo:
! TestFailed("The bayes object's 'wordinfo' did not compare the same at the end of all this!")
! if bayes.probcache != original_bayes.probcache:
! TestFailed("The bayes object's 'probcache' did not compare the same at the end of all this!")
!
! spam_msg.Delete()
print "Created a Spam message, and saw it get filtered and trained."
--- 233,339 ----
nham = bayes.nham
original_bayes = copy.copy(driver.manager.classifier_data.bayes)
! # for each watch folder, create a spam message, and do the training thang
! for msf_watch, folder_watch in driver.GetWatchFolderGenerator():
! print "Performing Spam test on watch folder '%s'..." % msf_watch.GetFQName()
! # Create a spam message in the Inbox - it should get immediately filtered
! msg, words = driver.CreateTestMessageInFolder(SPAM, folder_watch)
! # sleep to ensure filtering.
WaitForFilters()
+ # It should no longer be in the Inbox.
+ driver.CheckMessageFilteredFrom(folder_watch)
+ # It should be in the "sure is spam" folder.
spam_msg = driver.FindTestMessage(driver.folder_spam)
if spam_msg is None:
! TestFailed("The test message vanished from the Inbox, but didn't appear in Spam")
! # Check that none of the above caused training.
! if nspam != bayes.nspam:
! TestFailed("Something caused a new spam message to appear")
if nham != bayes.nham:
! TestFailed("Something caused a new ham message to appear")
! check_words(words, bayes, 0, 0)
!
! # Now move the message back to the inbox - it should get trained.
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! import train
if train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as ham yet")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as spam yet")
! spam_msg.Move(folder_watch)
! WaitForFilters()
! spam_msg = driver.FindTestMessage(folder_watch)
if spam_msg is None:
! TestFailed("The message appears to have been filtered out of the watch folder")
store_msg = driver.manager.message_store.GetMessage(spam_msg)
! need_untrain = True
! try:
! if nspam != bayes.nspam:
! TestFailed("There were not the same number of spam messages after a re-train")
! if nham+1 != bayes.nham:
! TestFailed("There was not one more ham messages after a re-train")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should not have been trained as spam yet")
! if not train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been trained as ham now")
! # word infos should have one extra ham
! check_words(words, bayes, 0, 1)
! # Now move it back to the Spam folder.
! # This should see the message un-trained as ham, and re-trained as Spam
! spam_msg.Move(driver.folder_spam)
! WaitForFilters()
! spam_msg = driver.FindTestMessage(driver.folder_spam)
! if spam_msg is None:
! TestFailed("Could not find the message in the Spam folder")
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! if nspam +1 != bayes.nspam:
! TestFailed("There should be one more spam now")
! if nham != bayes.nham:
! TestFailed("There should be the same number of hams again")
! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been trained as spam by now")
! if train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("This new spam message should have been un-trained as ham")
! # word infos should have one extra spam, no extra ham
! check_words(words, bayes, 1, 0)
! # Move the message to another folder, and make sure we still
! # identify it correctly as having been trained.
! # Move to the "unsure" folder, just cos we know about it, and
! # we know that no special watching of this folder exists.
! spam_msg.Move(driver.folder_unsure)
! spam_msg = driver.FindTestMessage(driver.folder_unsure)
! if spam_msg is None:
! TestFailed("Could not find the message in the Unsure folder")
! store_msg = driver.manager.message_store.GetMessage(spam_msg)
! if not train.been_trained_as_spam(store_msg, driver.manager.classifier_data):
! TestFailed("Message was not identified as Spam after moving")
!
! # word infos still be 'spam'
! check_words(words, bayes, 1, 0)
!
! # Now undo the damage we did.
! was_spam = train.untrain_message(store_msg, driver.manager.classifier_data)
! if not was_spam:
! TestFailed("Untraining this message did not indicate it was spam")
! if train.been_trained_as_spam(store_msg, driver.manager.classifier_data) or \
! train.been_trained_as_ham(store_msg, driver.manager.classifier_data):
! TestFailed("Untraining this message kept it has ham/spam")
! need_untrain = False
! finally:
! if need_untrain:
! train.untrain_message(store_msg, driver.manager.classifier_data)
!
! # Check all the counts are back where we started.
! if nspam != bayes.nspam:
! TestFailed("Spam count didn't get back to the same")
! if nham != bayes.nham:
! TestFailed("Ham count didn't get back to the same")
! check_words(words, bayes, 0, 0)
!
! if bayes.wordinfo != original_bayes.wordinfo:
! TestFailed("The bayes object's 'wordinfo' did not compare the same at the end of all this!")
! if bayes.probcache != original_bayes.probcache:
! TestFailed("The bayes object's 'probcache' did not compare the same at the end of all this!")
!
! spam_msg.Delete()
print "Created a Spam message, and saw it get filtered and trained."
***************
*** 340,345 ****
WaitForFilters()
# It should still be in the Inbox.
! if driver.FindTestMessage(folder1) is None:
! TestFailed("The test ham message appeared to have been filtered!")
# Manually move it to folder2
--- 352,356 ----
WaitForFilters()
# It should still be in the Inbox.
! driver.CheckMessageFilteredFrom(folder1)
# Manually move it to folder2
***************
*** 381,395 ****
def TestUnsureFilter(driver):
# Create a spam message in the Inbox - it should get immediately filtered
! msg, words = driver.CreateTestMessageInFolder(UNSURE, driver.folder_watch)
! # sleep to ensure filtering.
! WaitForFilters()
! # It should no longer be in the Inbox.
! if driver.FindTestMessage(driver.folder_watch) is not None:
! TestFailed("The test unsure message appeared to not be filtered")
! # It should be in the "unsure" folder.
! spam_msg = driver.FindTestMessage(driver.folder_unsure)
! if spam_msg is None:
! TestFailed("The test message vanished from the Inbox, but didn't appear in Unsure")
! spam_msg.Delete()
print "Created an unsure message, and saw it get filtered"
--- 392,407 ----
def TestUnsureFilter(driver):
# Create a spam message in the Inbox - it should get immediately filtered
! for msf_watch, folder_watch in driver.GetWatchFolderGenerator():
! print "Performing Spam test on watch folder '%s'..." % msf_watch.GetFQName()
! msg, words = driver.CreateTestMessageInFolder(UNSURE, folder_watch)
! # sleep to ensure filtering.
! WaitForFilters()
! # It should no longer be in the Inbox.
! driver.CheckMessageFilteredFrom(folder_watch)
! # It should be in the "unsure" folder.
! spam_msg = driver.FindTestMessage(driver.folder_unsure)
! if spam_msg is None:
! TestFailed("The test message vanished from the Inbox, but didn't appear in Unsure")
! spam_msg.Delete()
print "Created an unsure message, and saw it get filtered"
***************
*** 549,571 ****
checkpoint)
# message moved after we have ID, but before opening.
! folder = driver.folder_watch
! if is_ham:
! msg, words = driver.CreateTestMessageInFolder(HAM, folder)
! else:
! msg, words = driver.CreateTestMessageInFolder(SPAM, folder)
! try:
! _setup_for_mapi_failure(checkpoint, hr)
try:
! # sleep to ensure filtering.
! WaitForFilters()
finally:
! _restore_mapi_failure()
! if driver.FindTestMessage(folder) is None:
! TestFailed("We appear to have filtered a message even though we forced 'not found' failure")
! finally:
! print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr),
! checkpoint)
! if msg is not None:
! msg.Delete()
def do_failure_tests(manager):
--- 561,584 ----
checkpoint)
# message moved after we have ID, but before opening.
! for msf, folder in driver.GetWatchFolderGenerator():
! print "Testing in folder '%s'" % msf.GetFQName()
! if is_ham:
! msg, words = driver.CreateTestMessageInFolder(HAM, folder)
! else:
! msg, words = driver.CreateTestMessageInFolder(SPAM, folder)
try:
! _setup_for_mapi_failure(checkpoint, hr)
! try:
! # sleep to ensure filtering.
! WaitForFilters()
! finally:
! _restore_mapi_failure()
! if driver.FindTestMessage(folder) is None:
! TestFailed("We appear to have filtered a message even though we forced 'not found' failure")
finally:
! if msg is not None:
! msg.Delete()
! print "<- Finished MAPI error '%s' in %s" % (mapiutil.GetScodeString(hr),
! checkpoint)
def do_failure_tests(manager):
***************
*** 620,628 ****
assert not manager.test_suite_running, "already running??"
manager.test_suite_running = True
- run_nonfilter_tests(manager)
- # filtering tests take alot of time - do them last.
run_filter_tests(manager)
run_failure_tests(manager)
run_invalid_id_tests(manager)
print "*" * 20
print "Test suite finished without error!"
--- 633,641 ----
assert not manager.test_suite_running, "already running??"
manager.test_suite_running = True
run_filter_tests(manager)
run_failure_tests(manager)
run_invalid_id_tests(manager)
+ # non-filter tests take alot of time - do them last.
+ run_nonfilter_tests(manager)
print "*" * 20
print "Test suite finished without error!"
From anadelonbrin at users.sourceforge.net Mon Sep 8 01:04:19 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 8 03:04:24 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv14300/scripts
Modified Files:
sb_imapfilter.py
Log Message:
Fix two bugs found by Rafael Scholl.
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_imapfilter.py 5 Sep 2003 01:25:27 -0000 1.2
--- sb_imapfilter.py 8 Sep 2003 07:04:16 -0000 1.3
***************
*** 486,492 ****
response = imap.uid("SEARCH", "UNDELETED")
self._check(response, "SEARCH UNDELETED")
! if response[1][0] == "":
return []
- return response[1][0].split(' ')
def __getitem__(self, key):
--- 486,493 ----
response = imap.uid("SEARCH", "UNDELETED")
self._check(response, "SEARCH UNDELETED")
! if response[1][0]:
! return response[1][0].split(' ')
! else:
return []
def __getitem__(self, key):
From anadelonbrin at users.sourceforge.net Mon Sep 8 01:04:20 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 8 03:04:28 2003
Subject: [Spambayes-checkins] spambayes/spambayes UserInterface.py, 1.23,
1.24
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv14300/spambayes
Modified Files:
UserInterface.py
Log Message:
Fix two bugs found by Rafael Scholl.
Index: UserInterface.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/UserInterface.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** UserInterface.py 1 Sep 2003 10:33:48 -0000 1.23
--- UserInterface.py 8 Sep 2003 07:04:17 -0000 1.24
***************
*** 654,662 ****
def onChangeopts(self, **parms):
if parms.has_key("how"):
if parms["how"] == "Save advanced options":
pmap = self.advanced_options_map
- else:
- pmap = self.parm_ini_map
del parms["how"]
html = self._getHTMLClone()
--- 654,661 ----
def onChangeopts(self, **parms):
+ pmap = self.parm_ini_map
if parms.has_key("how"):
if parms["how"] == "Save advanced options":
pmap = self.advanced_options_map
del parms["how"]
html = self._getHTMLClone()
From mhammond at users.sourceforge.net Mon Sep 8 01:51:54 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 03:51:58 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 tester.py,1.19,1.20
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv22578
Modified Files:
tester.py
Log Message:
Accidently lost a call to a specific incremental training test - reinstate
it (and a few more tweaks trying to get it all running on an IMAP inbox)
Index: tester.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/tester.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** tester.py 8 Sep 2003 02:35:46 -0000 1.19
--- tester.py 8 Sep 2003 07:51:51 -0000 1.20
***************
*** 50,54 ****
def WaitForFilters():
- import pythoncom
# Must wait longer than normal, so when run with a timer we still work.
filter_event.clear()
--- 50,53 ----
***************
*** 163,169 ****
if self.FindTestMessage(folder) is None:
break
! sleep(.5)
else:
! TestFailed("The test message remained in folder %r" % folder.Name)
def _CleanTestMessageFromFolder(self, folder):
--- 162,170 ----
if self.FindTestMessage(folder) is None:
break
! for j in range(10):
! sleep(.05)
else:
! ms_folder = self.manager.message_store.GetFolder(folder)
! TestFailed("The test message remained in folder '%s'" % ms_folder.GetFQName())
def _CleanTestMessageFromFolder(self, folder):
***************
*** 352,364 ****
WaitForFilters()
# It should still be in the Inbox.
! driver.CheckMessageFilteredFrom(folder1)
# Manually move it to folder2
msg.Move(folder2)
- # re-find it in folder2
- msg = driver.FindTestMessage(folder2)
-
# sleep to any processing in this folder.
WaitForFilters()
if nspam != bayes.nspam or nham != bayes.nham:
--- 353,367 ----
WaitForFilters()
# It should still be in the Inbox.
! if driver.FindTestMessage(folder1) is None:
! TestFailed("The test ham message appeared to have been filtered!")
# Manually move it to folder2
msg.Move(folder2)
# sleep to any processing in this folder.
WaitForFilters()
+ # re-find it in folder2
+ msg = driver.FindTestMessage(folder2)
+ if driver.FindTestMessage(folder2) is None:
+ TestFailed("Couldn't find the ham message we just moved")
if nspam != bayes.nspam or nham != bayes.nham:
***************
*** 383,386 ****
--- 386,390 ----
mgr.config.filter.watch_include_sub)
num = 0
+ folders = []
for f in gen:
print "Running ham filter tests on folder '%s'" % f.GetFQName()
***************
*** 388,391 ****
--- 392,406 ----
_DoTestHamFilter(driver, f)
num += 1
+ folders.append(f)
+ # Now test incremental train logic, between all these folders.
+ if len(folders)<2:
+ print "NOTE: Can't do incremental training tests as only 1 watch folder is in place"
+ else:
+ for f in folders:
+ # 'targets' is a list of all folders except this
+ targets = folders[:]
+ targets.remove(f)
+ for t in targets:
+ _DoTestHamTrain(driver, f, t)
print "Created a Ham message, and saw it remain in place (in %d watch folders.)" % num
***************
*** 614,621 ****
def filter_message_with_event(msg, mgr, all_actions=True):
import filter
! try:
! return filter._original_filter_message(msg, mgr, all_actions)
! finally:
! filter_event.set()
def test(manager):
--- 629,636 ----
def filter_message_with_event(msg, mgr, all_actions=True):
import filter
! ret = filter._original_filter_message(msg, mgr, all_actions)
! if ret != "Failed":
! filter_event.set() # only set if it works
! return ret
def test(manager):
From mhammond at users.sourceforge.net Mon Sep 8 01:53:08 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 03:53:12 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
dialogs.rc, 1.37, 1.38
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv22804
Modified Files:
dialogs.rc
Log Message:
Correct grammar on dialog.
Index: dialogs.rc
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v
retrieving revision 1.37
retrieving revision 1.38
diff -C2 -d -r1.37 -r1.38
*** dialogs.rc 5 Sep 2003 06:51:00 -0000 1.37
--- dialogs.rc 8 Sep 2003 07:53:05 -0000 1.38
***************
*** 42,46 ****
CONTROL "",IDC_DELAY2_SLIDER,"msctls_trackbar32",TBS_AUTOTICKS |
TBS_TOP | WS_TABSTOP,16,73,148,22
! LTEXT "Delay between new mail check",IDC_STATIC,16,62,101,8
EDITTEXT IDC_DELAY2_TEXT,165,79,40,14,ES_AUTOHSCROLL
LTEXT "seconds",IDC_STATIC,207,82,28,8
--- 42,46 ----
CONTROL "",IDC_DELAY2_SLIDER,"msctls_trackbar32",TBS_AUTOTICKS |
TBS_TOP | WS_TABSTOP,16,73,148,22
! LTEXT "Delay between processing items",IDC_STATIC,16,62,142,8
EDITTEXT IDC_DELAY2_TEXT,165,79,40,14,ES_AUTOHSCROLL
LTEXT "seconds",IDC_STATIC,207,82,28,8
From mhammond at users.sourceforge.net Mon Sep 8 01:53:49 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 03:53:52 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/installer
spambayes_addin.iss, 1.9, 1.10
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/installer
In directory sc8-pr-cvs1:/tmp/cvs-serv23184
Modified Files:
spambayes_addin.iss
Log Message:
Installed "welcome.html" into the wrong place
Index: spambayes_addin.iss
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** spambayes_addin.iss 4 Sep 2003 11:59:35 -0000 1.9
--- spambayes_addin.iss 8 Sep 2003 07:53:46 -0000 1.10
***************
*** 18,22 ****
Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver
Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
! Source: "dist\docs\welcome.html"; DestDir: "{app}"; Flags: isreadme
[UninstallDelete]
--- 18,22 ----
Source: "dist\spambayes_addin.dll"; DestDir: "{app}"; Flags: ignoreversion regserver
Source: "dist\*.*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs
! Source: "dist\docs\welcome.html"; DestDir: "{app}\docs"; Flags: isreadme
[UninstallDelete]
From mhammond at users.sourceforge.net Mon Sep 8 01:56:52 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 03:56:55 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.18,1.19
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv23584
Modified Files:
Version.py
Log Message:
Outlook moves to 0.8, and the download location changes to
http://spambayes.sourceforge.net/windows.html
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** Version.py 6 Sep 2003 04:03:35 -0000 1.18
--- Version.py 8 Sep 2003 07:56:50 -0000 1.19
***************
*** 31,38 ****
},
"Outlook" : {
! "Version": 0.7,
! "BinaryVersion": 0.7,
"Description": "SpamBayes Outlook Addin",
! "Date": "August 9, 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
"Full Description Binary":
--- 31,38 ----
},
"Outlook" : {
! "Version": 0.8,
! "BinaryVersion": 0.8,
"Description": "SpamBayes Outlook Addin",
! "Date": "September 8, 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
"Full Description Binary":
***************
*** 41,45 ****
# versions will still go to the new page.
# We may also like to have a "Release Notes Page" item later?
! "Download Page": "http://starship.python.net/crew/mhammond/spambayes"
},
"POP3 Proxy" : {
--- 41,45 ----
# versions will still go to the new page.
# We may also like to have a "Release Notes Page" item later?
! "Download Page": "http://spambayes.sourceforge.net/windows.html"
},
"POP3 Proxy" : {
From mhammond at users.sourceforge.net Mon Sep 8 02:17:31 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 04:17:37 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/installer
spambayes_addin.iss, 1.10, 1.11
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/installer
In directory sc8-pr-cvs1:/tmp/cvs-serv27220
Modified Files:
spambayes_addin.iss
Log Message:
Version 008
Index: spambayes_addin.iss
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** spambayes_addin.iss 8 Sep 2003 07:53:46 -0000 1.10
--- spambayes_addin.iss 8 Sep 2003 08:17:29 -0000 1.11
***************
*** 5,10 ****
[Setup]
AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.7
! AppVersion=0.7
DefaultDirName={pf}\Spambayes Outlook Addin
DefaultGroupName=Spambayes Outlook Addin
--- 5,10 ----
[Setup]
AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.8
! AppVersion=0.8
DefaultDirName={pf}\Spambayes Outlook Addin
DefaultGroupName=Spambayes Outlook Addin
From sjoerd at users.sourceforge.net Mon Sep 8 03:19:46 2003
From: sjoerd at users.sourceforge.net (Sjoerd Mullender)
Date: Mon Sep 8 05:19:52 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv5793
Modified Files:
sb_imapfilter.py
Log Message:
Trivial fix for IMAP over SSL.
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** sb_imapfilter.py 8 Sep 2003 07:04:16 -0000 1.3
--- sb_imapfilter.py 8 Sep 2003 09:19:44 -0000 1.4
***************
*** 111,115 ****
try:
if options["imap", "use_ssl"]:
! from imaplib import IMAP_SSL as BaseIMAP
else:
from imaplib import IMAP4 as BaseIMAP
--- 111,115 ----
try:
if options["imap", "use_ssl"]:
! from imaplib import IMAP4_SSL as BaseIMAP
else:
from imaplib import IMAP4 as BaseIMAP
From ta-meyer at ihug.co.nz Mon Sep 8 22:22:47 2003
From: ta-meyer at ihug.co.nz (Tony Meyer)
Date: Mon Sep 8 05:22:52 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13031B7CA6@its-xchg4.massey.ac.nz>
Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz>
> Trivial fix for IMAP over SSL.
[...]
> ! from imaplib import IMAP_SSL as BaseIMAP
[...]
> ! from imaplib import IMAP4_SSL as BaseIMAP
. Thanks. (it does say, "untested" )
=Tony Meyer
From mhammond at users.sourceforge.net Mon Sep 8 04:56:00 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 06:56:09 2003
Subject: [Spambayes-checkins]
website applications.ht, 1.21, 1.22 download.ht,
1.17, 1.18 index.ht, 1.24, 1.25 windows.ht, 1.26, 1.27
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv22912
Modified Files:
applications.ht download.ht index.ht windows.ht
Log Message:
Outlook Version 008.
Index: applications.ht
===================================================================
RCS file: /cvsroot/spambayes/website/applications.ht,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** applications.ht 5 Sep 2003 08:02:54 -0000 1.21
--- applications.ht 8 Sep 2003 10:55:58 -0000 1.22
***************
*** 28,33 ****
Availability
Mark has packaged together an installer for the plugin.
! Go to the Windows page for more.
! This is currently at version 007.
Download the alpha4 release.
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 28,32 ----
Availability
Mark has packaged together an installer for the plugin.
! Go to the Windows page for more.
Download the alpha4 release.
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
Index: download.ht
===================================================================
RCS file: /cvsroot/spambayes/website/download.ht,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** download.ht 5 Sep 2003 08:02:54 -0000 1.17
--- download.ht 8 Sep 2003 10:55:58 -0000 1.18
***************
*** 24,29 ****
Outlook Plugin
Mark has packaged up a standalone installer of the Outlook plugin. Get it
! from the Windows page on this site.
! This is currently at version 007.
Other
--- 24,28 ----
Outlook Plugin
Mark has packaged up a standalone installer of the Outlook plugin. Get it
! from the Windows page on this site.
Other
Index: index.ht
===================================================================
RCS file: /cvsroot/spambayes/website/index.ht,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** index.ht 4 Sep 2003 21:35:29 -0000 1.24
--- index.ht 8 Sep 2003 10:55:58 -0000 1.25
***************
*** 5,9 ****
News
! Outlook plugin release 007 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
Fifth pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
--- 5,9 ----
News
! Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
Fifth pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
Index: windows.ht
===================================================================
RCS file: /cvsroot/spambayes/website/windows.ht,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** windows.ht 13 Aug 2003 13:42:10 -0000 1.26
--- windows.ht 8 Sep 2003 10:55:58 -0000 1.27
***************
*** 5,23 ****
Spambayes on Windows
! Outlook 2000/Outlook XP
! If you are using Outlook 2000 or Outlook XP (not Outlook
! Express) you should be able to simply download and run the
! Outlook
! plug-in installer (alternate
! download site). A separate Python installation is not necessary.
! Mark's
! website has more details on this package.
! Note that all users who installed version 006 or earlier of the plugin are
! recommended to upgrade to the 007 release.
! The Outlook add-in works with all versions of Windows, and all
! versions of Outlook 2000 and later - see below for more details.
If you have a problem, the best way to help make progress is to look for
--- 5,29 ----
Spambayes on Windows
! This page has information for users of Microsoft Outlook,
! and for all other mail clients
!
Outlook 2000/Outlook XP
! Latest Release
! The latest release is Version 008 - see the
! release notes
! or download the installation program.
!
! General Information
!
! The Outlook addin is an application of the SpamBayes project. It works with
! Outlook 2000 and later on Windows 98 and later. It does not work with
! Outlook Express - see below for other options
!
! In general, you should download and install the latest version, as shown
! above. Older versions and the release notes can be viewed at the
! spambayes releases page.
!
If you have a problem, the best way to help make progress is to look for
***************
*** 28,40 ****
the
! troubleshooting.html file that is installed with the plugin.
!
! Compatibility
!
! As far as we know, the latest release of the plugin works with all
! versions of Windows and Outlook 2000 and later (Outlook 2003 should have
! the latest Technical Release installed). If you have a problem, please follow
! the bug reporting steps above; we will endeavour to keep this page
! up to date.
Installing the Outlook Client From Source
--- 34,38 ----
the
! troubleshooting.html file that is installed with the plugin.
Installing the Outlook Client From Source
***************
*** 67,73 ****
the win32all installer. Finally, double-click the addin.py script in the
Outlook2000 folder of the Spambayes distribution and you should be good to
! go.
! Other Mail Clients
If you use any other mail client on Windows, you currently need to
--- 65,71 ----
the win32all installer. Finally, double-click the addin.py script in the
Outlook2000 folder of the Spambayes distribution and you should be good to
! go. See the README.txt file in the Outlook2000 directory for more details
! Non Outlook Solutions
If you use any other mail client on Windows, you currently need to
From sjoerd at users.sourceforge.net Mon Sep 8 05:09:27 2003
From: sjoerd at users.sourceforge.net (Sjoerd Mullender)
Date: Mon Sep 8 07:09:31 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv25754/scripts
Modified Files:
sb_imapfilter.py
Log Message:
Count all messages being classified instead of just the ones from the last folder.
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** sb_imapfilter.py 8 Sep 2003 09:19:44 -0000 1.4
--- sb_imapfilter.py 8 Sep 2003 11:09:25 -0000 1.5
***************
*** 644,648 ****
if options["globals", "verbose"]:
t = time.time()
! count = None
# Select the spam folder and unsure folder to make sure they exist
--- 644,651 ----
if options["globals", "verbose"]:
t = time.time()
! count = {}
! count["ham"] = 0
! count["spam"] = 0
! count["unsure"] = 0
# Select the spam folder and unsure folder to make sure they exist
***************
*** 654,659 ****
imap.SelectFolder(filter_folder)
folder = IMAPFolder(filter_folder)
! count = folder.Filter(self.classifier, self.spam_folder,
self.unsure_folder)
if options["globals", "verbose"]:
--- 657,664 ----
imap.SelectFolder(filter_folder)
folder = IMAPFolder(filter_folder)
! subcount = folder.Filter(self.classifier, self.spam_folder,
self.unsure_folder)
+ for key in count.keys():
+ count[key] += subcount.get(key, 0)
if options["globals", "verbose"]:
From sjoerd at acm.org Mon Sep 8 14:09:19 2003
From: sjoerd at acm.org (Sjoerd Mullender)
Date: Mon Sep 8 07:09:36 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.3,1.4
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz>
References: <1ED4ECF91CDED24C8D012BCF2B034F130212AE53@its-xchg4.massey.ac.nz>
Message-ID: <20030908110919.BD41A74132@indus.ins.cwi.nl>
On Mon, Sep 8 2003 "Tony Meyer" wrote:
> > Trivial fix for IMAP over SSL.
> [...]
> > ! from imaplib import IMAP_SSL as BaseIMAP
> [...]
> > ! from imaplib import IMAP4_SSL as BaseIMAP
>
> . Thanks. (it does say, "untested" )
With this fix it does seem to be working.
However, I do see that messages get scored every time I run
sb_imapfilter -c.
Another checkin to improve things is on its way.
-- Sjoerd Mullender
From mhammond at users.sourceforge.net Mon Sep 8 06:01:46 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 8 08:01:50 2003
Subject: [Spambayes-checkins] website quotes.ht,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv3027
Modified Files:
quotes.ht
Log Message:
You would think I could spell my own name correctly
Index: quotes.ht
===================================================================
RCS file: /cvsroot/spambayes/website/quotes.ht,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** quotes.ht 4 Sep 2003 23:29:47 -0000 1.5
--- quotes.ht 8 Sep 2003 12:01:44 -0000 1.6
***************
*** 79,83 ****
I followed up too :) I have a working Outlook Addin in place.
! Mark Hammod
uttering words he is likely to regret for years to come
--- 79,83 ----
I followed up too :) I have a working Outlook Addin in place.
! Mark Hammond
uttering words he is likely to regret for years to come
From sjoerd at users.sourceforge.net Mon Sep 8 08:46:59 2003
From: sjoerd at users.sourceforge.net (Sjoerd Mullender)
Date: Mon Sep 8 10:47:06 2003
Subject: [Spambayes-checkins] spambayes/spambayes message.py,1.35,1.36
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv8250
Modified Files:
message.py
Log Message:
Use comparison with None for selve instance, since a shelve instance
can compare false if the shelve is empty.
I believe this fixes bug 801952 "Imapfilter appending headers".
Index: message.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v
retrieving revision 1.35
retrieving revision 1.36
diff -C2 -d -r1.35 -r1.36
*** message.py 5 Sep 2003 01:25:27 -0000 1.35
--- message.py 8 Sep 2003 14:46:57 -0000 1.36
***************
*** 123,131 ****
def store(self):
! if self.db:
self.db.sync()
def _getState(self, msg):
! if self.db:
try:
(msg.c, msg.t) = self.db[msg.getId()]
--- 123,131 ----
def store(self):
! if self.db is not None:
self.db.sync()
def _getState(self, msg):
! if self.db is not None:
try:
(msg.c, msg.t) = self.db[msg.getId()]
***************
*** 134,142 ****
def _setState(self, msg):
! if self.db:
self.db[msg.getId()] = (msg.c, msg.t)
def _delState(self, msg):
! if self.db:
del self.db[msg.getId()]
--- 134,142 ----
def _setState(self, msg):
! if self.db is not None:
self.db[msg.getId()] = (msg.c, msg.t)
def _delState(self, msg):
! if self.db is not None:
del self.db[msg.getId()]
***************
*** 273,278 ****
def addSBHeaders(self, prob, clues):
! '''Add hammie header, and remember message's classification. Also,
! add optional headers if needed.'''
if prob < options['Categorization','ham_cutoff']:
--- 273,278 ----
def addSBHeaders(self, prob, clues):
! """Add hammie header, and remember message's classification. Also,
! add optional headers if needed."""
if prob < options['Categorization','ham_cutoff']:
From mhammond at users.sourceforge.net Mon Sep 8 21:08:45 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:12:07 2003
Subject: [Spambayes-checkins] website windows.ht,1.27,1.28
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv6709
Modified Files:
windows.ht
Log Message:
008.1
Index: windows.ht
===================================================================
RCS file: /cvsroot/spambayes/website/windows.ht,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** windows.ht 8 Sep 2003 10:55:58 -0000 1.27
--- windows.ht 9 Sep 2003 03:08:42 -0000 1.28
***************
*** 11,17 ****
Latest Release
! The latest release is Version 008 - see the
release notes
! or download the installation program.
--- 11,17 ----
Latest Release
! The latest release is Version 008.1 - see the
release notes
! or download the installation program.
From mhammond at users.sourceforge.net Mon Sep 8 18:59:08 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:16:44 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
FolderSelector.py, 1.27, 1.28
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv19723
Modified Files:
FolderSelector.py
Log Message:
Handle message store exceptions when populating the list.
Index: FolderSelector.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v
retrieving revision 1.27
retrieving revision 1.28
diff -C2 -d -r1.27 -r1.28
*** FolderSelector.py 5 Sep 2003 06:54:23 -0000 1.27
--- FolderSelector.py 9 Sep 2003 00:59:06 -0000 1.28
***************
*** 266,271 ****
# Compare the eid of the stores, then the objects
CompareEntryIDs = self.manager.message_store.session.CompareEntryIDs
! return CompareEntryIDs(mapi.BinFromHex(id1[0]), mapi.BinFromHex(id2[0])) and \
! CompareEntryIDs(mapi.BinFromHex(id1[1]), mapi.BinFromHex(id2[1]))
def InIDs(self, id, ids):
--- 266,275 ----
# Compare the eid of the stores, then the objects
CompareEntryIDs = self.manager.message_store.session.CompareEntryIDs
! try:
! return CompareEntryIDs(mapi.BinFromHex(id1[0]), mapi.BinFromHex(id2[0])) and \
! CompareEntryIDs(mapi.BinFromHex(id1[1]), mapi.BinFromHex(id2[1]))
! except pythoncom.com_error:
! # invalid IDs are never the same
! return False
def InIDs(self, id, ids):
***************
*** 339,343 ****
folders_to_expand = []
for folder_id in self.selected_ids:
! folder = self.manager.message_store.GetFolder(folder_id)
while folder is not None:
parent = folder.GetParent()
--- 343,351 ----
folders_to_expand = []
for folder_id in self.selected_ids:
! try:
! folder = self.manager.message_store.GetFolder(folder_id)
! except self.manager.message_store.MsgStoreException, details:
! print "Can't find a folder to expand:", details
! folder = None
while folder is not None:
parent = folder.GetParent()
***************
*** 636,640 ****
mgr.dialog_parser = dialogs.LoadDialogs()
! ids = []
d=FolderSelector(0, mgr, ids, single_select = False)
if d.DoModal() != win32con.IDOK:
--- 644,648 ----
mgr.dialog_parser = dialogs.LoadDialogs()
! ids = [("0000","0000"),] # invalid ID for testing.
d=FolderSelector(0, mgr, ids, single_select = False)
if d.DoModal() != win32con.IDOK:
From mhammond at users.sourceforge.net Mon Sep 8 20:14:31 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:17:17 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
FolderSelector.py, 1.29, 1.30
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv31757
Modified Files:
FolderSelector.py
Log Message:
Get the "New Folder" button working again (*sigh*)
Index: FolderSelector.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -C2 -d -r1.29 -r1.30
*** FolderSelector.py 9 Sep 2003 01:26:12 -0000 1.29
--- FolderSelector.py 9 Sep 2003 02:14:28 -0000 1.30
***************
*** 194,205 ****
return array.array("c", buf), extra
def UnpackTVItem(buffer):
item_mask, item_hItem, item_state, item_stateMask, \
item_textptr, item_cchText, item_image, item_selimage, \
item_cChildren, item_param = struct.unpack("10i", buffer)
! # ensure only items listed by the mask are valid
if not (item_mask & commctrl.TVIF_TEXT): item_textptr = item_cchText = None
if not (item_mask & commctrl.TVIF_CHILDREN): item_cChildren = None
- if not (item_mask & commctrl.TVIF_HANDLE): item_hItem = None
if not (item_mask & commctrl.TVIF_IMAGE): item_image = None
if not (item_mask & commctrl.TVIF_PARAM): item_param = None
--- 194,228 ----
return array.array("c", buf), extra
+ # Make a new buffer suitable for querying hitem's attributes.
+ def EmptyTVITEM(hitem, mask = None, text_buf_size=512):
+ extra = [] # objects we must keep references to
+ if mask is None:
+ mask = commctrl.TVIF_HANDLE | commctrl.TVIF_STATE | commctrl.TVIF_TEXT | \
+ commctrl.TVIF_IMAGE | commctrl.TVIF_SELECTEDIMAGE | \
+ commctrl.TVIF_CHILDREN | commctrl.TVIF_PARAM
+ if mask & commctrl.TVIF_TEXT:
+ text_buffer = array.array("c", "\0" * text_buf_size)
+ extra.append(text_buffer)
+ text_addr, text_len = text_buffer.buffer_info()
+ else:
+ text_addr = text_len = 0
+ format = "iiiiiiiiii"
+ buf = struct.pack(format,
+ mask, hitem,
+ 0, 0,
+ text_addr, text_len, # text
+ 0, 0,
+ 0, 0)
+ return array.array("c", buf), extra
+
def UnpackTVItem(buffer):
item_mask, item_hItem, item_state, item_stateMask, \
item_textptr, item_cchText, item_image, item_selimage, \
item_cChildren, item_param = struct.unpack("10i", buffer)
! # ensure only items listed by the mask are valid (except we assume the
! # handle is always valid - some notifications (eg, TVN_ENDLABELEDIT) set a
! # mask that doesn't include the handle, but the docs explicity say it is.)
if not (item_mask & commctrl.TVIF_TEXT): item_textptr = item_cchText = None
if not (item_mask & commctrl.TVIF_CHILDREN): item_cChildren = None
if not (item_mask & commctrl.TVIF_IMAGE): item_image = None
if not (item_mask & commctrl.TVIF_PARAM): item_param = None
***************
*** 359,364 ****
def _GetTVItem(self, h):
! text_buffer = "\0" * 1024
! buffer, extra = PackTVITEM(h, 0, 0, text_buffer, None, None, None, -1)
win32gui.SendMessage(self.list, commctrl.TVM_GETITEM,
0, buffer.buffer_info()[0])
--- 382,386 ----
def _GetTVItem(self, h):
! buffer, extra = EmptyTVITEM(h)
win32gui.SendMessage(self.list, commctrl.TVM_GETITEM,
0, buffer.buffer_info()[0])
From xenogeist at users.sourceforge.net Mon Sep 8 19:07:28 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Tue Sep 9 01:18:21 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv21080/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Update the tray program to use the new script names.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** pop3proxy_tray.py 2 Sep 2003 20:12:13 -0000 1.5
--- pop3proxy_tray.py 9 Sep 2003 01:07:26 -0000 1.6
***************
*** 33,38 ****
sys.path.insert(-1, os.getcwd())
sys.path.insert(-1, os.path.dirname(os.getcwd()))
! import pop3proxy
from spambayes import Dibbler
from spambayes.Options import options
--- 33,39 ----
sys.path.insert(-1, os.getcwd())
sys.path.insert(-1, os.path.dirname(os.getcwd()))
+ sys.path.insert(-1, os.path.join(os.path.dirname(os.getcwd()),"scripts"))
! import sb_server
from spambayes import Dibbler
from spambayes.Options import options
***************
*** 73,83 ****
# Get the custom icon
! startedIconPathName = "%s\\windows\\resources\\sb-started.ico" % \
! (os.path.dirname(pop3proxy.__file__),)
! stoppedIconPathName = "%s\\windows\\resources\\sb-stopped.ico" % \
! (os.path.dirname(pop3proxy.__file__),)
# When 1.0a6 is released, the above line will need to change to:
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
! ## (os.path.dirname(pop3proxy.__file__),)
if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
--- 74,84 ----
# Get the custom icon
! startedIconPathName = "%s\\..\\windows\\resources\\sb-started.ico" % \
! (os.path.dirname(sb_server.__file__),)
! stoppedIconPathName = "%s\\..\\windows\\resources\\sb-stopped.ico" % \
! (os.path.dirname(sb_server.__file__),)
# When 1.0a6 is released, the above line will need to change to:
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
! ## (os.path.dirname(sb_server.__file__),)
if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
***************
*** 93,101 ****
self.tip = None
! # Start up pop3proxy
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
! # XXX start that, and if not kick pop3proxy off in a separate thread.
! pop3proxy.prepare(state=pop3proxy.state)
self.StartStop()
--- 94,102 ----
self.tip = None
! # Start up sb_server
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
! # XXX start that, and if not kick sb_server off in a separate thread.
! sb_server.prepare(state=sb_server.state)
self.StartStop()
***************
*** 105,110 ****
#%i spam %i unsure %i session %i active
tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
! (pop3proxy.state.numSpams, pop3proxy.state.numHams, pop3proxy.state.numUnsure,
! pop3proxy.state.totalSessions, pop3proxy.state.activeSessions)
else:
tip = "SpamBayes is not running"
--- 106,111 ----
#%i spam %i unsure %i session %i active
tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
! (sb_server.state.numSpams, sb_server.state.numHams, sb_server.state.numUnsure,
! sb_server.state.totalSessions, sb_server.state.activeSessions)
else:
tip = "SpamBayes is not running"
***************
*** 167,171 ****
def OnExit(self):
if self.started:
! pop3proxy.stop(pop3proxy.state)
self.started = False
DestroyWindow(self.hwnd)
--- 168,172 ----
def OnExit(self):
if self.started:
! sb_server.stop(sb_server.state)
self.started = False
DestroyWindow(self.hwnd)
***************
*** 173,178 ****
def StartProxyThread(self):
! args = (pop3proxy.state,)
! thread.start_new_thread(pop3proxy.start, args)
self.started = True
--- 174,179 ----
def StartProxyThread(self):
! args = (sb_server.state,)
! thread.start_new_thread(sb_server.start, args)
self.started = True
***************
*** 180,187 ****
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
! # XXX start/stop that, and if not kick pop3proxy off in a separate
# XXX thread, or stop the thread that was started.
if self.started:
! pop3proxy.stop(pop3proxy.state)
self.started = False
self.control_functions[START_STOP_ID] = ("Start SpamBayes",
--- 181,188 ----
# XXX This needs to be finished off.
# XXX This should determine if we are using the service, and if so
! # XXX start/stop that, and if not kick sb_server off in a separate
# XXX thread, or stop the thread that was started.
if self.started:
! sb_server.stop(sb_server.state)
self.started = False
self.control_functions[START_STOP_ID] = ("Start SpamBayes",
From mhammond at users.sourceforge.net Mon Sep 8 19:26:14 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:23:16 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
FolderSelector.py, 1.28, 1.29
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv23849
Modified Files:
FolderSelector.py
Log Message:
Another place we used to check for GetFolder() returning None, which it
never does any more. Still a couple of places that are unguarded, but
they would have previously failed with a None return anyway, so are left
alone.
Index: FolderSelector.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/FolderSelector.py,v
retrieving revision 1.28
retrieving revision 1.29
diff -C2 -d -r1.28 -r1.29
*** FolderSelector.py 9 Sep 2003 00:59:06 -0000 1.28
--- FolderSelector.py 9 Sep 2003 01:26:12 -0000 1.29
***************
*** 91,106 ****
temp_id = mapi.HexFromBin(store_eid), mapi.HexFromBin(eid)
try:
child_folder = manager.message_store.GetFolder(temp_id)
! if child_folder is not None:
! spec = FolderSpec(child_folder.GetID(), name, folder_spec.ignore_eids)
! # If we have no children at all, indicate
! # the item is not expandable.
! table = child_folder.OpenEntry().GetHierarchyTable(0)
! if table.GetRowCount(0) == 0:
! spec.children = []
! else:
! spec.children = None # Flag as "not yet built"
! children.append(spec)
! except pythoncom.com_error, details:
# Users have reported failure here - it is not clear if the
# entire tree is going to fail, or just this folder
--- 91,108 ----
temp_id = mapi.HexFromBin(store_eid), mapi.HexFromBin(eid)
try:
+ # may get MsgStoreException for GetFolder, or
+ # a mapi exception for the underlying MAPI stuff we then call.
+ # Either way, just skip it.
child_folder = manager.message_store.GetFolder(temp_id)
! spec = FolderSpec(child_folder.GetID(), name, folder_spec.ignore_eids)
! # If we have no children at all, indicate
! # the item is not expandable.
! table = child_folder.OpenEntry().GetHierarchyTable(0)
! if table.GetRowCount(0) == 0:
! spec.children = []
! else:
! spec.children = None # Flag as "not yet built"
! children.append(spec)
! except (pythoncom.com_error, manager.message_store.MsgStoreException), details:
# Users have reported failure here - it is not clear if the
# entire tree is going to fail, or just this folder
From xenogeist at users.sourceforge.net Mon Sep 8 19:26:18 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Tue Sep 9 01:23:24 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.10,
1.11
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv24014/windows
Modified Files:
pop3proxy_service.py
Log Message:
Update the windows service to use the new script names.
Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** pop3proxy_service.py 31 Aug 2003 02:17:46 -0000 1.10
--- pop3proxy_service.py 9 Sep 2003 01:26:16 -0000 1.11
***************
*** 1,3 ****
! # Run the pop3proxy as a WinNT service. Should work on Windows 2000
# and Windows XP.
#
--- 1,3 ----
! # Run the sb_server as a WinNT service. Should work on Windows 2000
# and Windows XP.
#
***************
*** 26,29 ****
--- 26,31 ----
this_filename=__file__
except NameError:
+ this_filename = sys.argv[0]
+ if not os.path.isabs(sys.argv[0]):
# Python 2.3 __main__
# patch up sys.argv, as our cwd will confuse service registration code
***************
*** 31,37 ****
--- 33,42 ----
this_filename = sys.argv[0]
+ print sys.argv[0]
sb_dir = os.path.dirname(os.path.dirname(this_filename))
+ sb_scripts_dir = os.path.join(sb_dir,"scripts")
sys.path.insert(0, sb_dir)
+ sys.path.insert(-1, sb_scripts_dir)
# and change directory here, so pop3proxy uses the default
# config file etc
***************
*** 44,48 ****
# The spambayes imports we need.
! import pop3proxy
# The win32 specific modules.
--- 49,53 ----
# The spambayes imports we need.
! import sb_server
# The win32 specific modules.
***************
*** 68,73 ****
class Service(win32serviceutil.ServiceFramework):
_svc_name_ = "pop3proxy"
! _svc_display_name_ = "SpamBayes pop3proxy Service"
_svc_deps_ = ['tcpip'] # We depend on the tcpip service.
def __init__(self, args):
--- 73,80 ----
class Service(win32serviceutil.ServiceFramework):
+ # The script name was changed to "sb_server" but I'll leave this as pop3proxy
+ # overwise people might accidently run two proxies.
_svc_name_ = "pop3proxy"
! _svc_display_name_ = "SpamBayes Service"
_svc_deps_ = ['tcpip'] # We depend on the tcpip service.
def __init__(self, args):
***************
*** 80,90 ****
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
self.event_stopping.set()
! pop3proxy.stop(pop3proxy.state)
def SvcDoRun(self):
# Setup our state etc
! pop3proxy.prepare(state=pop3proxy.state)
! assert not pop3proxy.state.launchUI, "Service can't launch a UI"
# Start the thread running the server.
--- 87,97 ----
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
self.event_stopping.set()
! sb_server.stop(sb_server.state)
def SvcDoRun(self):
# Setup our state etc
! sb_server.prepare(state=sb_server.state)
! assert not sb_server.state.launchUI, "Service can't launch a UI"
# Start the thread running the server.
***************
*** 122,126 ****
# Write another event log record.
! s = pop3proxy.state
status = " after %d sessions (%d ham, %d spam, %d unsure)" % \
(s.totalSessions, s.numHams, s.numSpams, s.numUnsure)
--- 129,133 ----
# Write another event log record.
! s = sb_server.state
status = " after %d sessions (%d ham, %d spam, %d unsure)" % \
(s.totalSessions, s.numHams, s.numSpams, s.numUnsure)
***************
*** 135,139 ****
try:
try:
! pop3proxy.start(pop3proxy.state)
except SystemExit:
# user requested shutdown
--- 142,146 ----
try:
try:
! sb_server.start(sb_server.state)
except SystemExit:
# user requested shutdown
From mhammond at users.sourceforge.net Mon Sep 8 21:09:51 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:23:33 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.19,1.20
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6900
Modified Files:
Version.py
Log Message:
0.81 of Outlook.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** Version.py 8 Sep 2003 07:56:50 -0000 1.19
--- Version.py 9 Sep 2003 03:09:49 -0000 1.20
***************
*** 31,38 ****
},
"Outlook" : {
! "Version": 0.8,
! "BinaryVersion": 0.8,
"Description": "SpamBayes Outlook Addin",
! "Date": "September 8, 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
"Full Description Binary":
--- 31,38 ----
},
"Outlook" : {
! "Version": 0.81,
! "BinaryVersion": 0.81,
"Description": "SpamBayes Outlook Addin",
! "Date": "September 9, 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
"Full Description Binary":
From mhammond at users.sourceforge.net Mon Sep 8 19:13:10 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 01:23:38 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs
opt_processors.py, 1.13, 1.14
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv21813
Modified Files:
opt_processors.py
Log Message:
Handle message store exception in the folder ID processor.
Index: opt_processors.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/opt_processors.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** opt_processors.py 4 Sep 2003 11:59:05 -0000 1.13
--- opt_processors.py 9 Sep 2003 01:13:07 -0000 1.14
***************
*** 318,329 ****
for eid in ids:
if eid is not None:
! folder = mgr.message_store.GetFolder(eid)
! if folder is None:
! name = ""
! else:
if self.use_fqn:
name = folder.GetFQName()
else:
name = folder.name
names.append(name)
win32gui.SetWindowText(self.GetControl(), self.name_joiner.join(names))
--- 318,329 ----
for eid in ids:
if eid is not None:
! try:
! folder = mgr.message_store.GetFolder(eid)
if self.use_fqn:
name = folder.GetFQName()
else:
name = folder.name
+ except mgr.message_store.MsgStoreException:
+ name = ""
names.append(name)
win32gui.SetWindowText(self.GetControl(), self.name_joiner.join(names))
From anadelonbrin at users.sourceforge.net Mon Sep 8 18:36:57 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 01:34:31 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv15856/scripts
Modified Files:
sb_imapfilter.py
Log Message:
Another bug found by Rafael Scholl - some imap servers don't put the quotation mark
around folder names like they should (according to the RFC), and we stripped them off
whether they were there or not, which meant that some names would be missing the
first and last characters in the folder list. Fix this.
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** sb_imapfilter.py 8 Sep 2003 11:09:25 -0000 1.5
--- sb_imapfilter.py 9 Sep 2003 00:36:54 -0000 1.6
***************
*** 268,274 ****
# the case
if self.folder_delimiter == ',':
! print """WARNING: Your imap server uses commas as the folder
! delimiter. This may cause unpredictable errors."""
! folders.append(fol[m.end()+5:-1])
folders.sort()
return folders
--- 268,275 ----
# the case
if self.folder_delimiter == ',':
! print "WARNING: Your imap server uses a comma as the " \
! "folder delimiter. This may cause unpredictable " \
! "errors."
! folders.append(fol[m.end()+4:].strip('"'))
folders.sort()
return folders
From anadelonbrin at users.sourceforge.net Tue Sep 9 00:05:17 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 02:05:21 2003
Subject: [Spambayes-checkins] spambayes/spambayes ProxyUI.py,1.20,1.21
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27936/spambayes
Modified Files:
ProxyUI.py
Log Message:
A bug report had 'text' ending up as None, which stoped the messages displaying.
This handles that error, so that things keep working, at least, but it would be good
to figure out how it is that 'text' can become None. As far as I can see, it means
that get_payload() is returning None, but I don't think it's meant to be able to do that.
Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** ProxyUI.py 1 Sep 2003 06:07:27 -0000 1.20
--- ProxyUI.py 9 Sep 2003 06:05:15 -0000 1.21
***************
*** 489,492 ****
--- 489,494 ----
if type(text) == type([]): # gotta be a 'right' way to do this
text = "(this message is a digest of %s messages)" % (len(text))
+ elif text is None:
+ text = "(this message has no body)"
else:
text = text.replace(' ', ' ') # Else they'll be quoted
From mhammond at users.sourceforge.net Tue Sep 9 00:17:36 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 9 02:17:39 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.28,1.29
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv29384
Modified Files:
storage.py
Log Message:
Add a close method to the various storage classes.
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.28
retrieving revision 1.29
diff -C2 -d -r1.28 -r1.29
*** storage.py 5 Sep 2003 01:15:28 -0000 1.28
--- storage.py 9 Sep 2003 06:17:33 -0000 1.29
***************
*** 141,144 ****
--- 141,148 ----
fp.close()
+ def close(self):
+ # we keep no reasources open - nothing to do
+ pass
+
# Values for our changed words map
WORD_DELETED = "D"
***************
*** 157,160 ****
--- 161,172 ----
self.load()
+ def close(self):
+ # Close our underlying database. Better not assume all databases
+ # have close functions!
+ def noop(): pass
+ getattr(self.db, "close", noop)()
+ getattr(self.dbm, "close", noop)()
+ # should not be a need to drop the 'dbm' or 'db' attributes.
+
def load(self):
'''Load state from database'''
***************
*** 292,295 ****
--- 304,313 ----
self.db_name = db_name
self.load()
+
+ def close(self):
+ '''Release all database resources'''
+ # As we (presumably) aren't as constrained as we are by file locking,
+ # don't force sub-classes to override
+ pass
def load(self):
From anadelonbrin at users.sourceforge.net Tue Sep 9 00:51:02 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 02:51:06 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv1951/scripts
Modified Files:
sb_server.py
Log Message:
sb_server should store and close the db before reopening it. gdbm fails if we do not
do this.
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_server.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_server.py 9 Sep 2003 06:51:00 -0000 1.2
***************
*** 718,721 ****
--- 718,726 ----
del proxyListeners[:]
+ # Close the database; we should anyway, and gdbm complains if we
+ # try to reopen it without closing it first.
+ state.bayes.store()
+ state.bayes.close()
+
prepare(state)
_createProxies(state.servers, state.proxyPorts)
From anadelonbrin at users.sourceforge.net Tue Sep 9 01:03:57 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 03:03:59 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.71,1.72
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6536/spambayes
Modified Files:
Options.py
Log Message:
Implement half of [ 801699 ] migration from hammie to sb_* scripts
(although in a different way)
This makes it optional to write the X-Spambayes-Trained header, to ease transition
from hammie.py
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.71
retrieving revision 1.72
diff -C2 -d -r1.71 -r1.72
*** Options.py 6 Sep 2003 04:03:35 -0000 1.71
--- Options.py 9 Sep 2003 07:03:54 -0000 1.72
***************
*** 569,572 ****
--- 569,579 ----
HEADER_NAME, RESTORE),
+ ("include_trained", "Add trained header", True,
+ """sb_mboxtrain.py can add a header that details how a message was
+ trained, which lets you keep track of it, and appropriately
+ re-train messages. However, if you would rather mboxtrain didn't
+ rewrite the message files, you can disable this option.""",
+ BOOLEAN, RESTORE),
+
("trained_header_name", "Trained header name", "X-Spambayes-Trained",
"""When training on a message, the name of the header to add with how
From anadelonbrin at users.sourceforge.net Tue Sep 9 01:03:57 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 03:04:04 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv6536/scripts
Modified Files:
sb_mboxtrain.py
Log Message:
Implement half of [ 801699 ] migration from hammie to sb_* scripts
(although in a different way)
This makes it optional to write the X-Spambayes-Trained header, to ease transition
from hammie.py
Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_mboxtrain.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_mboxtrain.py 9 Sep 2003 07:03:54 -0000 1.2
***************
*** 116,119 ****
--- 116,121 ----
continue
trained += 1
+ if not options["Headers", "include_trained"]:
+ continue
f = file(tfn, "wb")
f.write(msg.as_string())
***************
*** 155,177 ****
if msg_train(h, msg, is_spam, force):
trained += 1
! # Write it out with the Unix "From " line
! outf.write(msg.as_string(True))
! outf.seek(0)
! try:
! os.ftruncate(f.fileno(), 0)
! f.seek(0)
! except:
! # If anything goes wrong, don't try to write
! print "Problem truncating mbox--nothing written"
! raise
! try:
! for line in outf.xreadlines():
! f.write(line)
! except:
! print >> sys.stderr ("Problem writing mbox! Sorry, "
! "I tried my best, but your mail "
! "may be corrupted.")
! raise
fcntl.lockf(f, fcntl.LOCK_UN)
f.close()
--- 157,181 ----
if msg_train(h, msg, is_spam, force):
trained += 1
! if not options["Headers", "include_trained"]:
! # Write it out with the Unix "From " line
! outf.write(msg.as_string(True))
! if not options["Headers", "include_trained"]:
! outf.seek(0)
! try:
! os.ftruncate(f.fileno(), 0)
! f.seek(0)
! except:
! # If anything goes wrong, don't try to write
! print "Problem truncating mbox--nothing written"
! raise
! try:
! for line in outf.xreadlines():
! f.write(line)
! except:
! print >> sys.stderr ("Problem writing mbox! Sorry, "
! "I tried my best, but your mail "
! "may be corrupted.")
! raise
fcntl.lockf(f, fcntl.LOCK_UN)
f.close()
***************
*** 203,206 ****
--- 207,212 ----
msg_train(h, msg, is_spam, force)
trained += 1
+ if not options["Headers", "include_trained"]:
+ continue
f = file(tfn, "wb")
f.write(msg.as_string())
***************
*** 290,293 ****
--- 296,300 ----
if loud: print "Training ham (%s):" % g
train(h, g, False, force, trainnew, removetrained)
+ sys.stdout.flush()
save = True
***************
*** 295,298 ****
--- 302,306 ----
if loud: print "Training spam (%s):" % s
train(h, s, True, force, trainnew, removetrained)
+ sys.stdout.flush()
save = True
From anadelonbrin at users.sourceforge.net Tue Sep 9 01:44:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 03:44:06 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.51,1.52
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12808
Modified Files:
README.txt
Log Message:
Update documentation to new script names. Procmail section needs finishing.
Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.51
retrieving revision 1.52
diff -C2 -d -r1.51 -r1.52
*** README.txt 25 Aug 2003 08:55:19 -0000 1.51
--- README.txt 9 Sep 2003 07:43:59 -0000 1.52
***************
*** 67,71 ****
If you get your mail from a POP3 server, then all you should need to do
to get running is change your mail client to send and receive mail from
! "localhost", and then run "python pop3proxy.py -b" in the directory you
expanded the SpamBayes source into. This will open a web browser window
- click the "Configuration" link at the top right and fill in the various
--- 67,71 ----
If you get your mail from a POP3 server, then all you should need to do
to get running is change your mail client to send and receive mail from
! "localhost", and then run "python sb_server.py -b" in the directory you
expanded the SpamBayes source into. This will open a web browser window
- click the "Configuration" link at the top right and fill in the various
***************
*** 113,117 ****
Now launch pop3proxy, either by running the "pop3proxy_service.py" script
(for those using Windows 2000, Windows NT or Windows XP), or the
! "pop3proxy.py" script (for everyone else).
All you need to do to configure SpamBayes is to open a web page to
--- 113,117 ----
Now launch pop3proxy, either by running the "pop3proxy_service.py" script
(for those using Windows 2000, Windows NT or Windows XP), or the
! "sb_server.py" script (for everyone else).
All you need to do to configure SpamBayes is to open a web page to
***************
*** 138,145 ****
-----------
! To configure SpamBayes, run the "imapfilter.py" script, and open a web page
! to , click on the "Configuration" link at the top
! right, and fill in the relevant details. Everything should be ok with the
! defaults, except for the server information at the top.
You now need to let SpamBayes know which IMAP folders it should work with.
--- 138,145 ----
-----------
! To configure SpamBayes, run the "sb_imapfilter.py" script, and open a web
! page to , click on the "Configuration" link at the
! top right, and fill in the relevant details. Everything should be ok with
! the defaults, except for the server information at the top.
You now need to let SpamBayes know which IMAP folders it should work with.
***************
*** 154,158 ****
You then need to set the IMAP filter up to run periodically. At the moment,
you'll need to do this from a command (or DOS) prompt. You should run the
! command "python imapfilter.py -c -t -l 5". The '-c' means that the script
should classify new mail, the '-t' means that the script should train any
mail that you have told it to, and the '-l 5' means that the script should
--- 154,158 ----
You then need to set the IMAP filter up to run periodically. At the moment,
you'll need to do this from a command (or DOS) prompt. You should run the
! command "python sb_imapfilter.py -c -t -l 5". The '-c' means that the script
should classify new mail, the '-t' means that the script should train any
mail that you have told it to, and the '-l 5' means that the script should
***************
*** 167,171 ****
Procmail is straightforward.
! First, create a SpamBayes database, by running "hammiefilter.py -n". If
you have some mail around that you can use to train it, do you (see the
"command line training" section below). Note that if you don't, all your
--- 167,171 ----
Procmail is straightforward.
! First, create a SpamBayes database, by running "sb_filter.py -n". If
you have some mail around that you can use to train it, do you (see the
"command line training" section below). Note that if you don't, all your
***************
*** 187,191 ****
:0 fw:hamlock
! | /usr/local/bin/hammie.py -f -d -p $HOME/hammie.db
The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter
--- 187,191 ----
:0 fw:hamlock
! | /usr/local/bin/hammie.py -f -d -p $HOME/.hammie.db
The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter
***************
*** 199,203 ****
The result of running hammie.py in filter mode is that Procmail will use the
output from the run as the mail message for further processing downstream.
! Hammie.py inserts an X-SpamBayes-Classification header in the output message
which looks like:
--- 199,203 ----
The result of running hammie.py in filter mode is that Procmail will use the
output from the run as the mail message for further processing downstream.
! hammie.py inserts an X-SpamBayes-Classification header in the output message
which looks like:
From anadelonbrin at users.sourceforge.net Tue Sep 9 01:53:58 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 03:54:02 2003
Subject: [Spambayes-checkins] spambayes/spambayes OptionsClass.py,1.12,1.13
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv14538/spambayes
Modified Files:
OptionsClass.py
Log Message:
Fix for [ 802347 ] multiline options saved incorrectly
Index: OptionsClass.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/OptionsClass.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** OptionsClass.py 5 Sep 2003 01:15:28 -0000 1.12
--- OptionsClass.py 9 Sep 2003 07:53:56 -0000 1.13
***************
*** 436,440 ****
out.write(optname)
out.write(vi)
! out.write(self.unconvert(sectname, optname))
out.write('\n')
written.append((sectname, optname))
--- 436,441 ----
out.write(optname)
out.write(vi)
! newval = self.unconvert(sectname, optname)
! out.write(newval.replace("\n", "\n\t"))
out.write('\n')
written.append((sectname, optname))
***************
*** 465,469 ****
out.write(opt)
out.write(vi)
! out.write(self.unconvert(sect, opt))
out.write('\n')
written.append((sect, opt))
--- 466,471 ----
out.write(opt)
out.write(vi)
! newval = self.unconvert(sect, opt)
! out.write(newval.replace("\n", "\n\t"))
out.write('\n')
written.append((sect, opt))
From anadelonbrin at users.sourceforge.net Tue Sep 9 02:07:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 04:07:18 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_imapfilter.py,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv16641/scripts
Modified Files:
sb_imapfilter.py
Log Message:
Fix for [ 802545 ] crash when loggin off imapfilter UI
Index: sb_imapfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_imapfilter.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** sb_imapfilter.py 9 Sep 2003 00:36:54 -0000 1.6
--- sb_imapfilter.py 9 Sep 2003 08:07:10 -0000 1.7
***************
*** 199,202 ****
--- 199,203 ----
self.current_folder = None
self.do_expunge = do_expunge
+ self.logged_in = False
def login(self, username, pwd):
***************
*** 210,218 ****
else:
raise
def logout(self):
# sign off
if self.do_expunge:
! self.expunge()
BaseIMAP.logout(self) # superclass logout
--- 211,232 ----
else:
raise
+ self.logged_in = True
def logout(self):
# sign off
if self.do_expunge:
! # we may never have logged in, in which case we do nothing
! if self.logged_in:
! # expunge messages from the spam and unsure folders
! for fol in ["spam_folder",
! "unsure_folder",]:
! self.select(options["imap", fol])
! self.expunge()
! # expunge messages from the ham and spam training folders
! for fol_list in ["ham_train_folders",
! "spam_train_folders",]:
! for fol in options["imap", fol_list]:
! self.select(fol)
! self.expunge()
BaseIMAP.logout(self) # superclass logout
From anadelonbrin at users.sourceforge.net Tue Sep 9 02:21:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 04:21:33 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.17,1.18
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv18743
Modified Files:
CHANGELOG.txt
Log Message:
Bring up to date.
Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** CHANGELOG.txt 1 Sep 2003 21:36:42 -0000 1.17
--- CHANGELOG.txt 9 Sep 2003 08:21:19 -0000 1.18
***************
*** 1,6 ****
--- 1,29 ----
[Note that all dates are in English, not American format - i.e. day/month/year]
+ Alpha Release 6
+ ===============
+ Tony Meyer 09/09/2003 Fix for [ 802545 ] crash when loggin off imapfilter UI
+ Tony Meyer 09/09/2003 Fix for [ 802347 ] multiline options saved incorrectly
+ Tony Meyer 09/09/2003 Implement half of [ 801699 ] migration from hammie to sb_* scripts (although in a different way)
+ Tony Meyer 09/09/2003 sb_server should store and close the db before reopening it. gdbm fails if we do not do this. Fix this.
+ Tony Meyer 09/09/2003 imapfilter: correctly handle imap servers that (wrongly) fail to put folder names in quotation marks
+ Mark Hammond 09/09/2003 Add a close method to the various storage classes.
+ Sjoerd Mullender 08/09/2003 Fix for [ 801952 ] Imapfilter appending headers
+ Sjoerd Mullender 08/09/2003 imapfilter: Count all messages being classified instead of just the ones from the last folder.
+ Sjoerd Mullender 08/09/2003 Trivial fix for IMAP over SSL.
+ Tony Meyer 08/09/2003 imapfilter: handle a folder name as a literal when presenting a list to choose from
+ Tony Meyer 08/09/2003 imapfilter: handle imap servers that don't pass a blank result line for an empty search
+ Mark Hammond 08/09/2003 Outlook: When we fail to add the 'Spam' field to a read-only store (eg, hotmail), complain less loudly.
+ Mark Hammond 08/09/2003 Fix [ 798362 ] Toolbar becomes unresponsive and must be recreated
+ Tony Meyer 06/09/2003 Add a missing file that's necessary to try out the sb_pop3dnd.py script, and the version and options necessary.
+ Tony Meyer 06/09/2003 Fix [ 800555 ] 1.0a5 release missing key outlook files.
+ Tony Meyer 05/09/2003 Remove backwards compat code for options, and update all (I hope) the remaining code that uses it.
+ Tony Meyer 05/09/2003 Move and rename all top-level scripts other than setup.py
+
Alpha Release 5
===============
+ Tony Meyer 03/09/2003 imapfilter: We would crash if we hadn't set anything up and didn't prompt for password.
+ Adam Walker 03/09/2003 pop3proxy_tray: Switch icons based on if the proxy is running or not. Provide some info found on the information page in the tooltip of the icon.
+ Tony Meyer 02/09/2003 Add a rough autoconfigure script, which will set up both spambayes and a mail client to use pop3proxy
Richie Hindle 01/09/2003 Fix [ 797890 ] "assert hamcount <= nham" problem.
Tony Meyer 01/09/2003 Fix [ spambayes-Bugs-796996 ] smtp server not started until restart.
From anadelonbrin at users.sourceforge.net Tue Sep 9 02:55:53 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 9 04:55:57 2003
Subject: [Spambayes-checkins] spambayes/windows autoconfigure.py,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv24726/windows
Modified Files:
autoconfigure.py
Log Message:
Fix Outlook Express configuration to use the current user's settings, which removes the need to figure out which registry key to use.
I accidently checked these in previously, but didn't notate them:
add support for pocomail (thanks to Romain Guy)
improve the move_to_next_port function to not use ports already taken by spambayes
(thanks to Romain Guy)
add support for autoconfiguring Pegasus Mail (not finished)
Index: autoconfigure.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/autoconfigure.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** autoconfigure.py 5 Sep 2003 01:15:29 -0000 1.4
--- autoconfigure.py 9 Sep 2003 08:55:51 -0000 1.5
***************
*** 8,11 ****
--- 8,12 ----
o Opera Mail (M2) (POP3/SMTP only)
o Outlook Express (POP3/SMTP only)
+ o PocoMail (POP3/SMTP only)
To do:
***************
*** 83,88 ****
except socket.error:
portStr = str(port)
! if options["pop3proxy", "listen_ports"].find(portStr) != -1 or \
! options["smtpproxy", "listen_ports"].find(portStr) != -1:
continue
else:
--- 84,89 ----
except socket.error:
portStr = str(port)
! if portStr in options["pop3proxy", "listen_ports"] or \
! portStr in options["smtpproxy", "listen_ports"]:
continue
else:
***************
*** 412,421 ****
# here.
! def configure_outlook_express(key):
"""Configure OE to use the SpamBayes POP3 and SMTP proxies, and
configure SpamBayes to proxy the servers that OE was connecting to."""
# OE stores its configuration in the registry, not a file.
!
! key = key + "\\Software\\Microsoft\\Internet Account Manager\\Accounts"
import win32api
--- 413,421 ----
# here.
! def configure_outlook_express():
"""Configure OE to use the SpamBayes POP3 and SMTP proxies, and
configure SpamBayes to proxy the servers that OE was connecting to."""
# OE stores its configuration in the registry, not a file.
! key = "Software\\Microsoft\\Internet Account Manager\\Accounts"
import win32api
***************
*** 429,433 ****
smtp_proxy = smtp_proxy_port
! reg = win32api.RegOpenKeyEx(win32con.HKEY_USERS, key)
account_index = 0
while True:
--- 429,433 ----
smtp_proxy = smtp_proxy_port
! reg = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER, key)
account_index = 0
while True:
***************
*** 435,446 ****
config = {}
try:
! subkey_name = "%s\\%s" % \
! (key, win32api.RegEnumKey(reg, account_index))
except win32api.error:
break
account_index += 1
index = 0
! subkey = win32api.RegOpenKeyEx(win32con.HKEY_USERS, subkey_name, 0,
! win32con.KEY_READ | win32con.KEY_SET_VALUE)
while True:
# Loop through all the keys
--- 435,448 ----
config = {}
try:
! subkey_name = "%s\\%s" % (key,
! win32api.RegEnumKey(reg,
! account_index))
except win32api.error:
break
account_index += 1
index = 0
! subkey = win32api.RegOpenKeyEx(win32con.HKEY_CURRENT_USER,
! subkey_name, 0, win32con.KEY_READ |
! win32con.KEY_SET_VALUE)
while True:
# Loop through all the keys
***************
*** 657,663 ****
if __name__ == "__main__":
pmail_ini_dir = "C:\\Program Files\\PMAIL\\MAIL\\ADMIN"
- # XXX This is my OE key = "S-1-5-21-95318837-410984162-318601546-13224"
- # XXX but I presume it's different for everyone? I'll have to check on
- # XXX another machine.
#configure_eudora(eudora_ini_dir)
#configure_mozilla(mozilla_ini_dir)
--- 659,662 ----
***************
*** 665,668 ****
#configure_outlook_express()
#configure_pocomail()
! configure_pegasus_mail(pmail_ini_dir)
pass
--- 664,667 ----
#configure_outlook_express()
#configure_pocomail()
! #configure_pegasus_mail(pmail_ini_dir)
pass
From montanaro at users.sourceforge.net Tue Sep 9 12:30:36 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Tue Sep 9 14:30:46 2003
Subject: [Spambayes-checkins] spambayes README.txt,1.52,1.53
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv13975
Modified Files:
README.txt
Log Message:
tweak info about training from command line and scoring via procmail
Index: README.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/README.txt,v
retrieving revision 1.52
retrieving revision 1.53
diff -C2 -d -r1.52 -r1.53
*** README.txt 9 Sep 2003 07:43:59 -0000 1.52
--- README.txt 9 Sep 2003 18:30:34 -0000 1.53
***************
*** 183,199 ****
Once you've trained SpamBayes on your
! collection of know ham and spam, you can use the hammie.py script to
classify incoming mail like so:
:0 fw:hamlock
! | /usr/local/bin/hammie.py -f -d -p $HOME/.hammie.db
! The above Procmail recipe tells it to run /usr/local/bin/hammie.py in filter
! mode (-f), and to use the training results stored in the dbm-style file
! ~/hammie.db. While hammie.py is runnning, Procmail uses the lock file
! hamlock to prevent multiple invocations from stepping on each others' toes.
! (It's not strictly necessary in this case since no files on-disk are
! modified, but Procmail will still complain if you don't specify a lock
! file.)
The result of running hammie.py in filter mode is that Procmail will use the
--- 183,199 ----
Once you've trained SpamBayes on your
! collection of know ham and spam, you can use the sb_filter.py script to
classify incoming mail like so:
:0 fw:hamlock
! | /usr/local/bin/sb_filter.py
! The above Procmail recipe tells it to run /usr/local/bin/sb_filter.py.
! Since no command line arguments are given, it relies on the options file
! specified by the BAYESCUSTOMIZE variable for all parameters. While
! sb_filter.py is runnning, Procmail uses the lock file hamlock to prevent
! multiple invocations from stepping on each others' toes. (It's not strictly
! necessary in this case since no files on-disk are modified, but Procmail
! will still complain if you don't specify a lock file.)
The result of running hammie.py in filter mode is that Procmail will use the
***************
*** 284,288 ****
containing nothing but ham, you can train Spambayes using a command like
! hammie.py -g ~/tmp/newham -s ~/tmp/newspam
The above command is command-line-centric (eg. unix, or Windows command
--- 284,288 ----
containing nothing but ham, you can train Spambayes using a command like
! sb_mboxtrain.py -g ~/tmp/newham -s ~/tmp/newspam
The above command is command-line-centric (eg. unix, or Windows command
From anadelonbrin at users.sourceforge.net Tue Sep 9 22:12:23 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 10 00:12:25 2003
Subject: [Spambayes-checkins] spambayes/spambayes hammie.py, 1.9,
1.10 storage.py, 1.29, 1.30
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv31097/spambayes
Modified Files:
hammie.py storage.py
Log Message:
A patch for the patched patched patch :)
Tidy the whole open storage thing up, and it should once again work for hammie with
a pickle. (mmm...ham and pickle...)
Index: hammie.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** hammie.py 5 Sep 2003 15:18:03 -0000 1.9
--- hammie.py 10 Sep 2003 04:12:20 -0000 1.10
***************
*** 257,261 ****
if needed), 'r' for read-only, 'w' for read-write.
"""
! return Hammie(storage.open_storage((filename, mode), useDB))
--- 257,261 ----
if needed), 'r' for read-only, 'w' for read-write.
"""
! return Hammie(storage.open_storage(filename, useDB, mode))
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -C2 -d -r1.29 -r1.30
*** storage.py 9 Sep 2003 06:17:33 -0000 1.29
--- storage.py 10 Sep 2003 04:12:20 -0000 1.30
***************
*** 637,641 ****
}
! def open_storage(data_source_name, useDB=True):
"""Return a storage object appropriate to the given parameters.
--- 637,641 ----
}
! def open_storage(data_source_name, useDB=True, mode=None):
"""Return a storage object appropriate to the given parameters.
***************
*** 648,653 ****
then a DBDictClassifier is used."""
if useDB:
! if (isinstance(data_source_name, types.StringTypes) and
! data_source_name.find('::') != -1):
db_type, rest = data_source_name.split('::', 1)
if _storage_types.has_key(db_type.lower()):
--- 648,652 ----
then a DBDictClassifier is used."""
if useDB:
! if data_source_name.find('::') != -1:
db_type, rest = data_source_name.split('::', 1)
if _storage_types.has_key(db_type.lower()):
***************
*** 661,666 ****
klass = PickledClassifier
try:
! if isinstance(data_source_name, type(())):
! return klass(*data_source_name)
return klass(data_source_name)
except dbmstorage.error, e:
--- 660,665 ----
klass = PickledClassifier
try:
! if mode is not None:
! return klass(data_source_name, mode)
return klass(data_source_name)
except dbmstorage.error, e:
***************
*** 668,675 ****
# We expect this to hit a fair few people, so warn them nicely,
# rather than just printing the trackback.
! print >> sys.stderr, """\
! You do not have a dbm module available to use. You need to either use a
! pickle (see the FAQ), use Python 2.3 (or above), or install a dbm module
! such as bsddb (see http://sf.net/projects/pybsddb)."""
import sys
sys.exit()
--- 667,674 ----
# We expect this to hit a fair few people, so warn them nicely,
# rather than just printing the trackback.
! print >> sys.stderr, "You do not have a dbm module available " \
! "to use. You need to either use a pickle (see the FAQ)" \
! ", use Python 2.3 (or above), or install a dbm module " \
! "such as bsddb (see http://sf.net/projects/pybsddb)."
import sys
sys.exit()
From anadelonbrin at users.sourceforge.net Tue Sep 9 22:33:19 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 10 00:33:26 2003
Subject: [Spambayes-checkins] spambayes/contrib nway.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/contrib
In directory sc8-pr-cvs1:/tmp/cvs-serv1738/contrib
Modified Files:
nway.py
Log Message:
Update names of scripts in comments/docstrings.
Index: nway.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/contrib/nway.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** nway.py 5 Sep 2003 01:15:28 -0000 1.3
--- nway.py 10 Sep 2003 04:33:17 -0000 1.4
***************
*** 24,31 ****
series of mboxtrain.py commands:
! mboxtrain.py -f -d python.db -s python -g music -g family -g cars
! mboxtrain.py -f -d music.db -g python -s music -g family -g cars
! mboxtrain.py -f -d family.db -g python -g music -s family -g cars
! mboxtrain.py -f -d cars.db -g python -g music -g family -s cars
You'd then compare messages using a %(prog)s command like this:
--- 24,31 ----
series of mboxtrain.py commands:
! sb_mboxtrain.py -f -d python.db -s python -g music -g family -g cars
! sb_mboxtrain.py -f -d music.db -g python -s music -g family -g cars
! sb_mboxtrain.py -f -d family.db -g python -g music -s family -g cars
! sb_mboxtrain.py -f -d cars.db -g python -g music -g family -s cars
You'd then compare messages using a %(prog)s command like this:
From anadelonbrin at users.sourceforge.net Tue Sep 9 22:33:19 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 10 00:33:31 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_client.py, 1.1,
1.2 sb_dbexpimp.py, 1.1, 1.2 sb_filter.py, 1.1,
1.2 sb_notesfilter.py, 1.1, 1.2 sb_server.py, 1.2, 1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv1738/scripts
Modified Files:
sb_client.py sb_dbexpimp.py sb_filter.py sb_notesfilter.py
sb_server.py
Log Message:
Update names of scripts in comments/docstrings.
Index: sb_client.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_client.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_client.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_client.py 10 Sep 2003 04:33:17 -0000 1.2
***************
*** 1,5 ****
#! /usr/bin/env python
! """A client for hammiesrv.
Just feed it your mail on stdin, and it spits out the same message
--- 1,5 ----
#! /usr/bin/env python
! """A client for sb_xmlrpcserver.py.
Just feed it your mail on stdin, and it spits out the same message
Index: sb_dbexpimp.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_dbexpimp.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_dbexpimp.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_dbexpimp.py 10 Sep 2003 04:33:17 -0000 1.2
***************
*** 1,5 ****
#! /usr/bin/env python
! """dbExpImp.py - Bayes database export/import
Classes:
--- 1,5 ----
#! /usr/bin/env python
! """sb_dbexpimp.py - Bayes database export/import
Classes:
***************
*** 41,45 ****
Usage:
! dbExpImp [options]
options:
--- 41,45 ----
Usage:
! sb_dbexpimp [options]
options:
***************
*** 60,81 ****
Export pickled mybayes.db into mybayes.db.export as a csv flat file
! dbExpImp -e -d mybayes.db -f mybayes.db.export
Import mybayes.eb.export into a new DBM mybayes.db
! dbExpImp -i -D mybayes.db -f mybayes.db.export
Export, then import (reorganize) new pickled mybayes.db
! dbExpImp -e -i -n -d mybayes.db -f mybayes.db.export
Convert a bayes database from pickle to DBM
! dbExpImp -e -d abayes.db -f abayes.export
! dbExpImp -i -D abayes.db -f abayes.export
Create a new database (newbayes.db) from two
databases (abayes.db, bbayes.db)
! dbExpImp -e -d abayes.db -f abayes.export
! dbExpImp -e -d bbayes.db -f bbayes.export
! dbExpImp -i -d newbayes.db -f abayes.export
! dbExpImp -i -m -d newbayes.db -f bbayes.export
To Do:
--- 60,81 ----
Export pickled mybayes.db into mybayes.db.export as a csv flat file
! sb_dbexpimp -e -d mybayes.db -f mybayes.db.export
Import mybayes.eb.export into a new DBM mybayes.db
! sb_dbexpimp -i -D mybayes.db -f mybayes.db.export
Export, then import (reorganize) new pickled mybayes.db
! sb_dbexpimp -e -i -n -d mybayes.db -f mybayes.db.export
Convert a bayes database from pickle to DBM
! sb_dbexpimp -e -d abayes.db -f abayes.export
! sb_dbexpimp -i -D abayes.db -f abayes.export
Create a new database (newbayes.db) from two
databases (abayes.db, bbayes.db)
! sb_dbexpimp -e -d abayes.db -f abayes.export
! sb_dbexpimp -e -d bbayes.db -f bbayes.export
! sb_dbexpimp -i -d newbayes.db -f abayes.export
! sb_dbexpimp -i -m -d newbayes.db -f bbayes.export
To Do:
***************
*** 264,266 ****
runImport(dbFN, useDBM, newDBM, flatFN)
else:
! print >>sys.stderr, __doc__
\ No newline at end of file
--- 264,266 ----
runImport(dbFN, useDBM, newDBM, flatFN)
else:
! print >>sys.stderr, __doc__
Index: sb_filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_filter.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_filter.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_filter.py 10 Sep 2003 04:33:17 -0000 1.2
***************
*** 7,11 ****
##
## :0 fw
! ## | hammiefilter.py
##
## Then, you can set up your MUA to pipe ham and spam to it, one at a
--- 7,11 ----
##
## :0 fw
! ## | sb_filter.py
##
## Then, you can set up your MUA to pipe ham and spam to it, one at a
Index: sb_notesfilter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_notesfilter.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_notesfilter.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_notesfilter.py 10 Sep 2003 04:33:17 -0000 1.2
***************
*** 1,5 ****
#! /usr/bin/env python
! '''notesfilter.py - Lotus Notes Spambayes interface.
Classes:
--- 1,5 ----
#! /usr/bin/env python
! '''sb_notesfilter.py - Lotus Notes Spambayes interface.
Classes:
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_server.py 9 Sep 2003 06:51:00 -0000 1.2
--- sb_server.py 10 Sep 2003 04:33:17 -0000 1.3
***************
*** 1,11 ****
#!/usr/bin/env python
! """A POP3 proxy that works with classifier.py, and adds a simple
X-Spambayes-Classification header (ham/spam/unsure) to each incoming
! email. You point pop3proxy at your POP3 server, and configure your
email client to collect mail from the proxy then filter on the added
header. Usage:
! pop3proxy.py [options] [ []]
is the name of your real POP3 server
is the port number of your real POP3 server, which
--- 1,16 ----
#!/usr/bin/env python
! """The primary server for SpamBayes.
!
! Currently serves the web interface, and any configured POP3 and SMTP
! proxies.
!
! The POP3 proxy works with classifier.py, and adds a simple
X-Spambayes-Classification header (ham/spam/unsure) to each incoming
! email. You point the proxy at your POP3 server, and configure your
email client to collect mail from the proxy then filter on the added
header. Usage:
! sb_server.py [options] [ []]
is the name of your real POP3 server
is the port number of your real POP3 server, which
From mhammond at users.sourceforge.net Wed Sep 10 01:33:42 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Wed Sep 10 03:33:46 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.22,1.23
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv30907
Modified Files:
about.html
Log Message:
Fix absolute references
Index: about.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** about.html 1 Sep 2003 07:27:31 -0000 1.22
--- about.html 10 Sep 2003 07:33:39 -0000 1.23
***************
*** 24,28 ****
To Spambayes
.
If you want to add the Spam field to your Outlook views, follow
these instructions.
If you need help configuring SpamBayes, see our .
If you want to add the Spam field to your Outlook views, follow
these instructions.
If you need help configuring SpamBayes, see our
Update of /cvsroot/spambayes/spambayes/Outlook2000/docs
In directory sc8-pr-cvs1:/tmp/cvs-serv31745
Modified Files:
welcome.html
Log Message:
Make references relative, and background= for the table to avoid mozilla
bug 167262
Index: welcome.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/welcome.html,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** welcome.html 29 Aug 2003 15:18:22 -0000 1.4
--- welcome.html 10 Sep 2003 07:38:51 -0000 1.5
***************
*** 8,15 ****
! |
!   |
! |
Welcome to SpamBayes
--- 8,21 ----
!
!
! |
! |
! |
!
!
Welcome to SpamBayes
***************
*** 107,112 ****
Outlook does not allow us to automatically add the spam score to your Outlook folder
! views - but you can do it
manually by following these instructions.
--- 113,117 ----
Outlook does not allow us to automatically add the spam score to your Outlook folder
! views - but you can do it
manually by following these instructions.
From mhammond at users.sourceforge.net Wed Sep 10 01:42:49 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Wed Sep 10 03:42:52 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 manager.py,1.86,1.87
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv32476
Modified Files:
manager.py
Log Message:
Call new new close() method on the classifier rather than hacking our own.
Index: manager.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/manager.py,v
retrieving revision 1.86
retrieving revision 1.87
diff -C2 -d -r1.86 -r1.87
*** manager.py 7 Sep 2003 23:42:19 -0000 1.86
--- manager.py 10 Sep 2003 07:42:45 -0000 1.87
***************
*** 172,175 ****
--- 172,177 ----
def open_bayes(self):
raise NotImplementedError
+ def close_bayes(self, bayes):
+ bayes.close()
class PickleStorageManager(BasicStorageManager):
***************
*** 177,182 ****
def open_bayes(self):
return bayes_storage.PickledClassifier(self.bayes_filename)
- def close_bayes(self, bayes):
- pass
def open_mdb(self):
return cPickle.load(open(self.mdb_filename, 'rb'))
--- 179,182 ----
***************
*** 196,202 ****
fname = self.bayes_filename.encode(filesystem_encoding)
return bayes_storage.DBDictClassifier(fname)
- def close_bayes(self, bayes):
- bayes.db.close()
- bayes.dbm.close()
def open_mdb(self):
fname = self.mdb_filename.encode(filesystem_encoding)
--- 196,199 ----
From anadelonbrin at users.sourceforge.net Wed Sep 10 02:24:26 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 10 04:24:36 2003
Subject: [Spambayes-checkins] website faq.txt,1.40,1.41
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv7485
Modified Files:
faq.txt
Log Message:
Update the FAQ to help users of v8 of the plug-in to find the data folder.
Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.40
retrieving revision 1.41
diff -C2 -d -r1.40 -r1.41
*** faq.txt 5 Sep 2003 09:21:02 -0000 1.40
--- faq.txt 10 Sep 2003 08:24:24 -0000 1.41
***************
*** 489,502 ****
back in to recover from a corrupted database, or for any other reason.
! This directory is located in the "Application Data" directory. On Windows
! NT/2000/XP, this will probably be:
C:\\Documents and Settings\\[username]\\Application Data\\Spambayes
! On other versions of Windows, this will probably be:
C:\\Windows\\Application Data\\Spambayes
! Note that this folder may be hidden.
--- 489,505 ----
back in to recover from a corrupted database, or for any other reason.
! This directory is located in the "Application Data" directory. If you have
! version 008 of the plug-in, or higher, you can locate this directory by
! using the `Show Data Folder` button on the `Advanced` tab of the main
! `SpamBayes` manager dialog. If you need to locate it by hand, on Windows
! NT/2000/XP, it will probably be:
C:\\Documents and Settings\\[username]\\Application Data\\Spambayes
! On other versions of Windows, it will probably be:
C:\\Windows\\Application Data\\Spambayes
! Note that the 'Application Data' folder may be hidden.
From montanaro at users.sourceforge.net Wed Sep 10 08:25:22 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Wed Sep 10 10:25:26 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.30,1.31
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv6711
Modified Files:
storage.py
Log Message:
indicate whether or not the classifier's __init__ function has a mode arg
and call appropriately
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.30
retrieving revision 1.31
diff -C2 -d -r1.30 -r1.31
*** storage.py 10 Sep 2003 04:12:20 -0000 1.30
--- storage.py 10 Sep 2003 14:25:19 -0000 1.31
***************
*** 631,638 ****
return repr(self.invalid_name)
! _storage_types = {"dbm" : DBDictClassifier,
! "pickle" : PickledClassifier,
! "pgsql" : PGClassifier,
! "mysql" : mySQLClassifier,
}
--- 631,640 ----
return repr(self.invalid_name)
! # values are classifier class and True if it accepts a mode
! # arg, False otherwise
! _storage_types = {"dbm" : (DBDictClassifier, True),
! "pickle" : (PickledClassifier, False),
! "pgsql" : (PGClassifier, False),
! "mysql" : (mySQLClassifier, False),
}
***************
*** 651,666 ****
db_type, rest = data_source_name.split('::', 1)
if _storage_types.has_key(db_type.lower()):
! klass = _storage_types[db_type.lower()]
data_source_name = rest
else:
raise NoSuchClassifierError(db_type)
else:
! klass = DBDictClassifier
else:
! klass = PickledClassifier
try:
! if mode is not None:
return klass(data_source_name, mode)
! return klass(data_source_name)
except dbmstorage.error, e:
if str(e) == "No dbm modules available!":
--- 653,669 ----
db_type, rest = data_source_name.split('::', 1)
if _storage_types.has_key(db_type.lower()):
! klass, supports_mode = _storage_types[db_type.lower()]
data_source_name = rest
else:
raise NoSuchClassifierError(db_type)
else:
! klass, supports_mode = _storage_types["dbm"]
else:
! klass, supports_mode = _storage_types["pickle"]
try:
! if supports_mode and mode is not None:
return klass(data_source_name, mode)
! else:
! return klass(data_source_name)
except dbmstorage.error, e:
if str(e) == "No dbm modules available!":
From richiehindle at users.sourceforge.net Wed Sep 10 15:08:26 2003
From: richiehindle at users.sourceforge.net (Richie Hindle)
Date: Wed Sep 10 17:08:30 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv2019
Modified Files:
storage.py
Log Message:
Bug 803501: Fix the "No dbm modules available" message to print
rather than crash. Here's a thought: if people would run their edits
before committing them, we'd have fewer bug reports. Just an idea.
Also prepended a newline to the error message, so that it doesn't
run into the end of the "Loading database..." message when running
sb_server.py.
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.31
retrieving revision 1.32
diff -C2 -d -r1.31 -r1.32
*** storage.py 10 Sep 2003 14:25:19 -0000 1.31
--- storage.py 10 Sep 2003 21:08:24 -0000 1.32
***************
*** 670,678 ****
# We expect this to hit a fair few people, so warn them nicely,
# rather than just printing the trackback.
! print >> sys.stderr, "You do not have a dbm module available " \
"to use. You need to either use a pickle (see the FAQ)" \
", use Python 2.3 (or above), or install a dbm module " \
"such as bsddb (see http://sf.net/projects/pybsddb)."
- import sys
sys.exit()
--- 670,677 ----
# We expect this to hit a fair few people, so warn them nicely,
# rather than just printing the trackback.
! print >> sys.stderr, "\nYou do not have a dbm module available " \
"to use. You need to either use a pickle (see the FAQ)" \
", use Python 2.3 (or above), or install a dbm module " \
"such as bsddb (see http://sf.net/projects/pybsddb)."
sys.exit()
From anthonybaxter at users.sourceforge.net Wed Sep 10 22:57:57 2003
From: anthonybaxter at users.sourceforge.net (Anthony Baxter)
Date: Thu Sep 11 00:58:00 2003
Subject: [Spambayes-checkins] website faq.txt,1.41,1.42
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv23622
Modified Files:
faq.txt
Log Message:
add a couple of notes about whitelisting:
1. Use your mailer to intercept the emails before spambayes
or
2. Send code.
Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.41
retrieving revision 1.42
diff -C2 -d -r1.41 -r1.42
*** faq.txt 10 Sep 2003 08:24:24 -0000 1.41
--- faq.txt 11 Sep 2003 04:57:55 -0000 1.42
***************
*** 1087,1090 ****
--- 1087,1100 ----
tools like SpamBayes fill.
+ If you really need whitelisting, consider implementing rules in your
+ mailer to intercept the messages before they're passed to spambayes.
+
+ None of the existing Spambayes developers have an interest in
+ implementing whitelisting - if it's something you feel really strongly
+ about, download the source and look into making a patch. If someone
+ implements whitelisting, including all the user interface side needed
+ to manage it, there's no reason it wouldn't be accepted into the source
+ code.
+
.. _related page: related.html
From anthonybaxter at users.sourceforge.net Thu Sep 11 00:57:44 2003
From: anthonybaxter at users.sourceforge.net (Anthony Baxter)
Date: Thu Sep 11 02:57:48 2003
Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,NONE,1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10913
Added Files:
NEWTRICKS.txt
Log Message:
a file to record ideas that have and haven't been tried.
--- NEW FILE: NEWTRICKS.txt ---
This file is for ideas that have or have not yet been tried, and the
results of these ideas.
- If the email parser was able to emit warnings about misformed MIME
that it encountered, we could turn this warnings into tokens.
- The numeric entity support looks like it will only support 8-bit
characters - an obvious problem is from wacky wide-char entities.
From anthonybaxter at users.sourceforge.net Thu Sep 11 01:04:25 2003
From: anthonybaxter at users.sourceforge.net (Anthony Baxter)
Date: Thu Sep 11 03:04:28 2003
Subject: [Spambayes-checkins] website background.ht,1.16,1.17
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv12193
Modified Files:
background.ht
Log Message:
ascribe blame where blame is due.
Thanks to Gary (and Tim) for updating my view of history.
Index: background.ht
===================================================================
RCS file: /cvsroot/spambayes/website/background.ht,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** background.ht 4 Aug 2003 02:49:59 -0000 1.16
--- background.ht 11 Sep 2003 07:04:22 -0000 1.17
***************
*** 136,144 ****
with an 'unsure' answer.
!
! These two central limit
! approaches were dropped after Tim, Gary and Rob Hooft produced a combining
! scheme using chi-squared probabilities. This is now the default combining
! scheme.
Chi-combining is similar to the central limit approaches, but it doesn't
have the annoying training problems that central limit approaches suffered
--- 136,149 ----
with an 'unsure' answer.
! The central limit approaches were dropped after Gary suggested a
! novel but well-founded combining scheme using
! chi-squared
! probabilities.
! Tim implemented it, including a unique routine to compute
! -2*sum(ln p_i) without logarithms and without underflow worries.
! Many, many testers reported excellent results. This is now the
! default combining scheme. Some end cases were improved when
! Rob Hooft discovered a cleaner way to combine the internal overall ham
! and spam scores, which improved detection of the "middle ground".
Chi-combining is similar to the central limit approaches, but it doesn't
have the annoying training problems that central limit approaches suffered
From anthonybaxter at users.sourceforge.net Thu Sep 11 01:33:51 2003
From: anthonybaxter at users.sourceforge.net (Anthony Baxter)
Date: Thu Sep 11 03:33:58 2003
Subject: [Spambayes-checkins] website faq.txt,1.42,1.43
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv17601
Modified Files:
faq.txt
Log Message:
consistent c12n of SpamBayes (that's what it's called on the front page
of the website)
Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.42
retrieving revision 1.43
diff -C2 -d -r1.42 -r1.43
*** faq.txt 11 Sep 2003 04:57:55 -0000 1.42
--- faq.txt 11 Sep 2003 07:33:48 -0000 1.43
***************
*** 1,4 ****
====================================
! Spambayes Frequently Asked Questions
====================================
--- 1,4 ----
====================================
! SpamBayes Frequently Asked Questions
====================================
***************
*** 17,24 ****
This is a work in progress. Please feel free to ask questions and/or
! provide answers; For help with Spambayes, send email to the `Spambayes
mailing list`_.
! .. _Spambayes mailing list: mailto:spambayes@python.org
--- 17,24 ----
This is a work in progress. Please feel free to ask questions and/or
! provide answers; For help with SpamBayes, send email to the `SpamBayes
mailing list`_.
! .. _SpamBayes mailing list: mailto:spambayes@python.org
***************
*** 26,39 ****
========
! What is Spambayes?
------------------
! Spambayes is a tool used to segregate unwanted mail (spam) from the mail you
! want (ham). Before Spambayes can be your spam filter of choice you need to
train it on representative samples of email you receive. After it's been
! trained, you use Spambayes to classify new mail according to its spamminess
and hamminess qualities.
! To train Spambayes (which you don't need to do if you're going to be using
the POP3 proxy to classify messages, but you'll get better results from the
outset if you do) you need to save your incoming email for awhile,
--- 26,39 ----
========
! What is SpamBayes?
------------------
! SpamBayes is a tool used to segregate unwanted mail (spam) from the mail you
! want (ham). Before SpamBayes can be your spam filter of choice you need to
train it on representative samples of email you receive. After it's been
! trained, you use SpamBayes to classify new mail according to its spamminess
and hamminess qualities.
! To train SpamBayes (which you don't need to do if you're going to be using
the POP3 proxy to classify messages, but you'll get better results from the
outset if you do) you need to save your incoming email for awhile,
***************
*** 42,46 ****
and the nature of what spam looks like change over time. Once you've
collected a fair portion of each (anything is better than nothing, but it
! helps to have a couple hundred of each), you can tell Spambayes, "Here's my
ham and my spam". It will then process that mail and save information about
different patterns which appear in ham and spam. That information is then
--- 42,46 ----
and the nature of what spam looks like change over time. Once you've
collected a fair portion of each (anything is better than nothing, but it
! helps to have a couple hundred of each), you can tell SpamBayes, "Here's my
ham and my spam". It will then process that mail and save information about
different patterns which appear in ham and spam. That information is then
***************
*** 48,52 ****
below for details.
! When Spambayes filters your email, it compares each unclassified message
against the information it saved from training and makes a decision about
whether it thinks the message qualifies as ham or spam, or if it's unsure
--- 48,52 ----
below for details.
! When SpamBayes filters your email, it compares each unclassified message
against the information it saved from training and makes a decision about
whether it thinks the message qualifies as ham or spam, or if it's unsure
***************
*** 54,58 ****
message, either by adding a header (X-Spambayes-Classification:
spam|ham|unsure), modifying the To: or Subject: headers, or adding a "Spam"
! field to the message. Depending on which Spambayes application you are
using, it may then filter this message for you, or you can set up your own
filters (to file away suspected spam into its own mail folder, for example).
--- 54,58 ----
message, either by adding a header (X-Spambayes-Classification:
spam|ham|unsure), modifying the To: or Subject: headers, or adding a "Spam"
! field to the message. Depending on which SpamBayes application you are
using, it may then filter this message for you, or you can set up your own
filters (to file away suspected spam into its own mail folder, for example).
***************
*** 87,103 ****
------------------------------------
! There are four mailing lists which support Spambayes:
! 1. The `Spambayes list`_ provides a place for users to get help with
! Spambayes or help other users.
! 2. The `Spambayes developers list`_ provides a forum for people
maintaining and improving the package.
! 3. The `Spambayes announcements list`_ is a low-volume list where
announcements about new releases are posted.
! 4. The `Spambayes checkins list`_ receives summaries of all the
! changes to the Spambayes software. This is generally only of interest to
developers.
--- 87,103 ----
------------------------------------
! There are four mailing lists which support SpamBayes:
! 1. The `SpamBayes list`_ provides a place for users to get help with
! SpamBayes or help other users.
! 2. The `SpamBayes developers list`_ provides a forum for people
maintaining and improving the package.
! 3. The `SpamBayes announcements list`_ is a low-volume list where
announcements about new releases are posted.
! 4. The `SpamBayes checkins list`_ receives summaries of all the
! changes to the SpamBayes software. This is generally only of interest to
developers.
***************
*** 113,123 ****
messages.
! .. _Spambayes list: http://mail.python.org/mailman/listinfo/spambayes
! .. _Spambayes developers list: http://mail.python.org/mailman/listinfo/spambayes-dev
! .. _Spambayes announcements list: http://mail.python.org/mailman/listinfo/spambayes-announce
! .. _Spambayes checkins list: http://mail.python.org/mailman/listinfo/spambayes-checkins
! What do I need to install Spambayes?
------------------------------------
--- 113,123 ----
messages.
! .. _SpamBayes list: http://mail.python.org/mailman/listinfo/spambayes
! .. _SpamBayes developers list: http://mail.python.org/mailman/listinfo/spambayes-dev
! .. _SpamBayes announcements list: http://mail.python.org/mailman/listinfo/spambayes-announce
! .. _SpamBayes checkins list: http://mail.python.org/mailman/listinfo/spambayes-checkins
! What do I need to install SpamBayes?
------------------------------------
***************
*** 139,146 ****
! Is there a high level summary that shows how Spambayes works?
-------------------------------------------------------------
! There are eight main components to the Spambayes system:
1. A database. Loosely speaking, this is a collection of words and
--- 139,146 ----
! Is there a high level summary that shows how SpamBayes works?
-------------------------------------------------------------
! There are eight main components to the SpamBayes system:
1. A database. Loosely speaking, this is a collection of words and
***************
*** 175,179 ****
+-----------------+ +------------+ +-------------+
! | Outlook Express | | Spambayes | | |
| (or similar) | <--> | POP3 proxy | <--> | POP3 server |
| | | | | |
--- 175,179 ----
+-----------------+ +------------+ +-------------+
! | Outlook Express | | SpamBayes | | |
| (or similar) | <--> | POP3 proxy | <--> | POP3 server |
| | | | | |
***************
*** 205,209 ****
+-----------------+ +------------+ +-------------+
! | Outlook Express | | Spambayes | | |
| (or similar) | <--> | SMTP proxy | <--> | SMTP server |
| | | | | |
--- 205,209 ----
+-----------------+ +------------+ +-------------+
! | Outlook Express | | SpamBayes | | |
| (or similar) | <--> | SMTP proxy | <--> | SMTP server |
| | | | | |
***************
*** 254,258 ****
as ham and folders that contain mail to train as spam, and the filter
does this for you. You can also designate folders to filter, along with
! a folder for messages Spambayes is unsure about, and a folder for
suspected spam. When new mail arrives, the filter will move mail to the
appropriate location (ham is left in the original folder).
--- 254,258 ----
as ham and folders that contain mail to train as spam, and the filter
does this for you. You can also designate folders to filter, along with
! a folder for messages SpamBayes is unsure about, and a folder for
suspected spam. When new mail arrives, the filter will move mail to the
appropriate location (ham is left in the original folder).
***************
*** 269,273 ****
As well as these components, there's also a whole pile of utility scripts,
test harnesses and so on - see README.txt and TESTING.txt in the
! spambayes distribution for more information.
--- 269,273 ----
As well as these components, there's also a whole pile of utility scripts,
test harnesses and so on - see README.txt and TESTING.txt in the
! SpamBayes distribution for more information.
***************
*** 276,280 ****
! Does Spambayes work with Outlook Express?
-----------------------------------------
--- 276,280 ----
! Does SpamBayes work with Outlook Express?
-----------------------------------------
***************
*** 297,306 ****
! What clients will Spambayes work with in general?
-------------------------------------------------
! Spambayes will work with most POP3 or IMAP compatible clients. How you
implement depends on your local architecture. Users with access to procmail
! can just write a recipe that invokes spambayes like this::
:0fw
--- 297,306 ----
! What clients will SpamBayes work with in general?
-------------------------------------------------
! SpamBayes will work with most POP3 or IMAP compatible clients. How you
implement depends on your local architecture. Users with access to procmail
! can just write a recipe that invokes SpamBayes like this::
:0fw
***************
*** 345,349 ****
Users limited to POP3/IMAP communications to the server can use the POP3_ or
! IMAP_ proxies which are part of the Spambayes source.
.. _POP3: http://spambayes.sf.net/applications.html#pop3
--- 345,349 ----
Users limited to POP3/IMAP communications to the server can use the POP3_ or
! IMAP_ proxies which are part of the SpamBayes source.
.. _POP3: http://spambayes.sf.net/applications.html#pop3
***************
*** 351,355 ****
! How do I configure Eudora for use with Spambayes?
-------------------------------------------------
--- 351,355 ----
! How do I configure Eudora for use with SpamBayes?
-------------------------------------------------
***************
*** 436,440 ****
! Do I have to have Python installed to use Spambayes with Outlook?
-----------------------------------------------------------------
--- 436,440 ----
! Do I have to have Python installed to use SpamBayes with Outlook?
-----------------------------------------------------------------
***************
*** 443,447 ****
! Will Spambayes work with Outlook 2000 connecting to an Exchange 2000 server?
----------------------------------------------------------------------------
--- 443,447 ----
! Will SpamBayes work with Outlook 2000 connecting to an Exchange 2000 server?
----------------------------------------------------------------------------
***************
*** 454,458 ****
alpha3 source releases and CVS). Details about how to enable this feature
can be found in the "configuration.html" file in the "docs" directory in
! the folder Spambayes was installed to. You can also set up mail that is
trained using the "Delete as spam" button to be marked as read, rather than
all mail classified as spam. However, you should also see the
--- 454,458 ----
alpha3 source releases and CVS). Details about how to enable this feature
can be found in the "configuration.html" file in the "docs" directory in
! the folder SpamBayes was installed to. You can also set up mail that is
trained using the "Delete as spam" button to be marked as read, rather than
all mail classified as spam. However, you should also see the
***************
*** 580,588 ****
register a hit on the spammer's server.
! Using Spambayes
===============
! How do I train Spambayes (web method)?
--------------------------------------
--- 580,588 ----
register a hit on the spammer's server.
! Using SpamBayes
===============
! How do I train SpamBayes (web method)?
--------------------------------------
***************
*** 606,610 ****
! How do I train Spambayes (forward/bounce method)?
-------------------------------------------------
--- 606,610 ----
! How do I train SpamBayes (forward/bounce method)?
-------------------------------------------------
***************
*** 626,635 ****
! How do I train Spambayes (command line method)?
-----------------------------------------------
Given a pair of Unix mailbox format files (each message starts with a line
which begins with 'From '), one containing nothing but spam and the other
! containing nothing but ham, you can train Spambayes using a command like::
hammie.py -g ~/tmp/newham -s ~/tmp/newspam
--- 626,635 ----
! How do I train SpamBayes (command line method)?
-----------------------------------------------
Given a pair of Unix mailbox format files (each message starts with a line
which begins with 'From '), one containing nothing but spam and the other
! containing nothing but ham, you can train SpamBayes using a command like::
hammie.py -g ~/tmp/newham -s ~/tmp/newspam
***************
*** 639,643 ****
! How do I train Spambayes (Outlook plugin)?
-----------------------------------------------
--- 639,643 ----
! How do I train SpamBayes (Outlook plugin)?
-----------------------------------------------
***************
*** 656,660 ****
Once a message has been [correctly] trained there is no need to keep it
! around. However, Spambayes' accuracy is dependent upon having a "sufficient"
sample from which to make its decisions. Therefore, most users retain a fair
amount of spam in the event that they may wish to rebuild the corpus from
--- 656,660 ----
Once a message has been [correctly] trained there is no need to keep it
! around. However, SpamBayes' accuracy is dependent upon having a "sufficient"
sample from which to make its decisions. Therefore, most users retain a fair
amount of spam in the event that they may wish to rebuild the corpus from
***************
*** 662,666 ****
Of course, this begs the question: "how much is enough?" That is where the
! "art" of spambayes meets the science. Some users keep as many as several
thousand [recent] spam (as well as a similar number of ham). That is not to
say that you won't have excellent results with a tenth (or less) of that
--- 662,666 ----
Of course, this begs the question: "how much is enough?" That is where the
! "art" of SpamBayes meets the science. Some users keep as many as several
thousand [recent] spam (as well as a similar number of ham). That is not to
say that you won't have excellent results with a tenth (or less) of that
***************
*** 669,673 ****
! Why did Spambayes mark this obvious spam "unsure"?
--------------------------------------------------
--- 669,673 ----
! Why did SpamBayes mark this obvious spam "unsure"?
--------------------------------------------------
***************
*** 689,696 ****
! Spambayes doesn't seem to catch much spam. What gives?
-------------------------------------------------------
! Initially, Spambayes will not be able to distinguish spams from hams. With
no training inputs, the classifier will simply mark everything unsure. Once
you start training the classifier on a representative set of spams and hams
--- 689,696 ----
! SpamBayes doesn't seem to catch much spam. What gives?
-------------------------------------------------------
! Initially, SpamBayes will not be able to distinguish spams from hams. With
no training inputs, the classifier will simply mark everything unsure. Once
you start training the classifier on a representative set of spams and hams
***************
*** 712,716 ****
should move incorrectly classified messages to their correct locations
and retrain (which may happen automatically, depending on how you're
! using Spambayes).
--- 712,716 ----
should move incorrectly classified messages to their correct locations
and retrain (which may happen automatically, depending on how you're
! using SpamBayes).
***************
*** 728,732 ****
! How do I configure Spambayes?
-----------------------------
--- 728,732 ----
! How do I configure SpamBayes?
-----------------------------
***************
*** 805,809 ****
SpamBayes looks for your configuration file in three places - if it can't
find it, then, obviously, your options will not be loaded. The first place
! that Spambayes checks is the environment variable BAYESCUSTOMIZE. You can
set this to the path of your configuration file, wherever it is, and it will
be loaded. You can also specify more than one file, separated by the
--- 805,809 ----
SpamBayes looks for your configuration file in three places - if it can't
find it, then, obviously, your options will not be loaded. The first place
! that SpamBayes checks is the environment variable BAYESCUSTOMIZE. You can
set this to the path of your configuration file, wherever it is, and it will
be loaded. You can also specify more than one file, separated by the
***************
*** 858,862 ****
-------------------------------------
! While Spambayes does an excellent job of classifying incoming mail, it is
only as good as the data on which it was trained. Here are some tips to
help you create a good training set:
--- 858,862 ----
-------------------------------------
! While SpamBayes does an excellent job of classifying incoming mail, it is
only as good as the data on which it was trained. Here are some tips to
help you create a good training set:
***************
*** 864,870 ****
* Don't use old mail. The characteristics of your email change over time,
sometimes subtly, sometimes dramatically, so it's best to use very recent
! mail to train Spambayes. If you've abandoned an email address in the past
because it was getting spammed heavily, there are probably some clues in
! mail sent to your old address which would bias Spambayes.
* Check and recheck your training collections. While you are manually
--- 864,870 ----
* Don't use old mail. The characteristics of your email change over time,
sometimes subtly, sometimes dramatically, so it's best to use very recent
! mail to train SpamBayes. If you've abandoned an email address in the past
because it was getting spammed heavily, there are probably some clues in
! mail sent to your old address which would bias SpamBayes.
* Check and recheck your training collections. While you are manually
***************
*** 874,878 ****
! Can Spambayes be used to perform n-way classification?
------------------------------------------------------
--- 874,878 ----
! Can SpamBayes be used to perform n-way classification?
------------------------------------------------------
***************
*** 881,885 ****
`_. A demonstration script which performs
n-way classification was also recently added to the ``contrib`` directory of
! the Spambayes CVS repository.
--- 881,885 ----
`_. A demonstration script which performs
n-way classification was also recently added to the ``contrib`` directory of
! the SpamBayes CVS repository.
***************
*** 1010,1014 ****
! Are there plans to develop a server-side spambayes solution?
------------------------------------------------------------
--- 1010,1014 ----
! Are there plans to develop a server-side SpamBayes solution?
------------------------------------------------------------
***************
*** 1018,1022 ****
on what is spam and what is not, then this might work for you, but otherwise
you really have to have individual databases for each user. Either way, you
! should be able to modify spambayes easily enough to fit into your setup.
Some people have in fact done this and have been kind enough to donate `notes
about how they have gone about it`_. If you also do this but in some other
--- 1018,1022 ----
on what is spam and what is not, then this might work for you, but otherwise
you really have to have individual databases for each user. Either way, you
! should be able to modify SpamBayes easily enough to fit into your setup.
Some people have in fact done this and have been kind enough to donate `notes
about how they have gone about it`_. If you also do this but in some other
***************
*** 1088,1094 ****
If you really need whitelisting, consider implementing rules in your
! mailer to intercept the messages before they're passed to spambayes.
! None of the existing Spambayes developers have an interest in
implementing whitelisting - if it's something you feel really strongly
about, download the source and look into making a patch. If someone
--- 1088,1094 ----
If you really need whitelisting, consider implementing rules in your
! mailer to intercept the messages before they're passed to SpamBayes.
! None of the existing SpamBayes developers have an interest in
implementing whitelisting - if it's something you feel really strongly
about, download the source and look into making a patch. If someone
***************
*** 1103,1112 ****
---------------------------------------
! If you're not a Spambayes developer simply send your corrections or proposed
! questions and answers to the `Spambayes developers mailing list`_. If you
are a developer you need a recent version of Docutils_ and the tools/html.py
script from that distribution must be in a directory on your PATH.
! .. _Spambayes developers mailing list: spambayes-dev@python.org
.. _Docutils: http://docutils.sf.net/
--- 1103,1112 ----
---------------------------------------
! If you're not a SpamBayes developer simply send your corrections or proposed
! questions and answers to the `SpamBayes developers mailing list`_. If you
are a developer you need a recent version of Docutils_ and the tools/html.py
script from that distribution must be in a directory on your PATH.
! .. _SpamBayes developers mailing list: spambayes-dev@python.org
.. _Docutils: http://docutils.sf.net/
From ta-meyer at ihug.co.nz Thu Sep 11 20:45:39 2003
From: ta-meyer at ihug.co.nz (Tony Meyer)
Date: Thu Sep 11 03:45:45 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F1303346327@its-xchg4.massey.ac.nz>
Message-ID: <1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz>
> Here's a thought: if people would
> run their edits before committing them, we'd have fewer bug
> reports. Just an idea.
To be fair, I tested that the error message printed out correctly when I
added it, and Skip probably checked that messages were going to stderr when
he added those, just not the specific case of having no dbm modules
available. It's a big ask to check that *every* part of the code still
works. This is really a case for a more comprehensive suite of tests, so
that we *can* check every part of the code still works.
I've checked in a unittest (in test_storage.py) that tests that we still
fail (correctly) when no dbm modules are available. This is my first ever
unittest, so I'd welcome any critiques.
=Tony Meyer
From anadelonbrin at users.sourceforge.net Thu Sep 11 01:47:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 11 03:47:18 2003
Subject: [Spambayes-checkins] spambayes/spambayes/test test_storage.py, 1.2,
1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv20097/spambayes/test
Modified Files:
test_storage.py
Log Message:
Add a unittest to check that we correctly fail when no dbm modules are available.
Index: test_storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_storage.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** test_storage.py 25 Jul 2003 05:17:22 -0000 1.2
--- test_storage.py 11 Sep 2003 07:47:13 -0000 1.3
***************
*** 142,145 ****
--- 142,167 ----
_StorageTestBase.tearDown(self)
+ def fail_open_best(self, *args):
+ from spambayes import dbmstorage
+ raise dbmstorage.error("No dbm modules available!")
+ def success(self, *args):
+ self.succeeded = True
+ def testNoDBMAvailable(self):
+ import tempfile
+ from spambayes.storage import open_storage
+ DBDictClassifier_load = DBDictClassifier.load
+ DBDictClassifier.load = self.fail_open_best
+ sys_exit = sys.exit
+ sys.exit = self.success
+ self.succeeded = False
+ db_name = tempfile.mktemp("nodbmtest")
+ s = open_storage(db_name, True)
+ DBDictClassifier.load = DBDictClassifier_load
+ sys.exit = sys_exit
+ if not self.succeeded:
+ self.fail()
+ if os.path.isfile(db_name):
+ os.remove(db_name)
+
def suite():
# We dont want our base class run
From anadelonbrin at users.sourceforge.net Thu Sep 11 01:47:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 11 03:47:58 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.19,1.20
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv20208
Modified Files:
setup.py
Log Message:
If the old files exist, offer to delete them. Based on the patches worked up on
spambayes-dev by Skip, Richie and me.
Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** setup.py 7 Jul 2003 00:09:27 -0000 1.19
--- setup.py 11 Sep 2003 07:47:52 -0000 1.20
***************
*** 1,4 ****
--- 1,6 ----
#!/usr/bin/env python
+ import os
+
import sys
if sys.version < '2.2':
***************
*** 25,64 ****
from spambayes import __version__
! setup(
! name='spambayes',
! version = __version__,
! description = "Spam classification system",
! author = "the spambayes project",
! author_email = "spambayes@python.org",
! url = "http://spambayes.sourceforge.net",
! scripts=['unheader.py',
! 'hammie.py',
! 'hammiecli.py',
! 'hammiesrv.py',
! 'hammiefilter.py',
! 'pop3proxy.py',
! 'smtpproxy.py',
! 'proxytee.py',
! 'dbExpImp.py',
! 'mboxtrain.py',
! 'imapfilter.py',
! 'notesfilter.py',
! ],
! packages = [
! 'spambayes',
! 'spambayes.resources',
! ],
! classifiers = [
! 'Development Status :: 4 - Beta',
! 'Environment :: Console',
! 'License :: OSI Approved :: Python Software Foundation License',
! 'Operating System :: POSIX',
! 'Operating System :: MacOS :: MacOS X',
! 'Operating System :: Microsoft :: Windows :: Windows 95/98/2000',
! 'Operating System :: Microsoft :: Windows :: Windows NT/2000',
! 'Programming Language :: Python',
! 'Intended Audience :: End Users/Desktop',
! 'Topic :: Communications :: Email :: Filters',
! 'Topic :: Communications :: Email :: Post-Office :: POP3',
]
! )
--- 27,106 ----
from spambayes import __version__
! import distutils.command.install_scripts
! parent = distutils.command.install_scripts.install_scripts
! class install_scripts(parent):
! old_scripts=[
! 'unheader.py',
! 'hammie.py',
! 'hammiecli.py',
! 'hammiesrv.py',
! 'hammiefilter.py',
! 'pop3proxy.py',
! 'smtpproxy.py',
! 'proxytee.py',
! 'dbExpImp.py',
! 'mboxtrain.py',
! 'imapfilter.py',
! 'notesfilter.py',
]
!
! def run(self):
! err = False
! for s in self.old_scripts:
! s = os.path.join(self.install_dir, s)
! if os.path.exists(s):
! print >> sys.stderr, "Error: old script", s, "still exists."
! err = True
! if err:
! print >>sys.stderr, "Do you want to delete these scripts? (y/n)"
! answer = raw_input("")
! if answer == "y":
! for s in self.old_scripts:
! s = os.path.join(self.install_dir, s)
! try:
! os.remove(s)
! print "Removed", s
! except OSError:
! pass
! return parent.run(self)
!
! setup(
! name='spambayes',
! version = __version__,
! description = "Spam classification system",
! author = "the spambayes project",
! author_email = "spambayes@python.org",
! url = "http://spambayes.sourceforge.net",
! cmdclass = {'build_py': install_scripts},
! scripts=['scripts/sb_client.py',
! 'scripts/sb_dbexpimp.py',
! 'scripts/sb_filter.py',
! 'scripts/sb_imapfilter.py',
! 'scripts/sb_mailsort.py',
! 'scripts/sb_mboxtrain.py',
! 'scripts/sb_notesfilter.py',
! 'scripts/sb_pop3dnd.py',
! 'scripts/sb_server.py',
! 'scripts/sb_smtpproxy.py',
! 'scripts/sb_unheader.py',
! 'scripts/sb_upload.py',
! 'scripts/sb_xmlrpcserver.py',
! ],
! packages = [
! 'spambayes',
! 'spambayes.resources',
! ],
! classifiers = [
! 'Development Status :: 4 - Beta',
! 'Environment :: Console',
! 'License :: OSI Approved :: Python Software Foundation License',
! 'Operating System :: POSIX',
! 'Operating System :: MacOS :: MacOS X',
! 'Operating System :: Microsoft :: Windows :: Windows 95/98/2000',
! 'Operating System :: Microsoft :: Windows :: Windows NT/2000',
! 'Programming Language :: Python',
! 'Intended Audience :: End Users/Desktop',
! 'Topic :: Communications :: Email :: Filters',
! 'Topic :: Communications :: Email :: Post-Office :: POP3',
! ],
! )
From anadelonbrin at users.sourceforge.net Thu Sep 11 01:58:36 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 11 03:58:40 2003
Subject: [Spambayes-checkins] website faq.txt,1.43,1.44 index.ht,1.25,1.26
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv22019
Modified Files:
faq.txt index.ht
Log Message:
Add a couple of commas, and fix an unterminated section in the FAQ.
Index: faq.txt
===================================================================
RCS file: /cvsroot/spambayes/website/faq.txt,v
retrieving revision 1.43
retrieving revision 1.44
diff -C2 -d -r1.43 -r1.44
*** faq.txt 11 Sep 2003 07:33:48 -0000 1.43
--- faq.txt 11 Sep 2003 07:58:34 -0000 1.44
***************
*** 741,745 ****
`pop3.example.com` to `localhost` (or to the name of the machine you're
running the proxy on) in your email client's setup, and do the same with
! your equivalent of `smtp.example.com". Hit "Get new email" and look at the
headers of the emails (send yourself an email if you don't have any!) -
there should be an X-Spambayes-Classification header there. It probably
--- 741,745 ----
`pop3.example.com` to `localhost` (or to the name of the machine you're
running the proxy on) in your email client's setup, and do the same with
! your equivalent of `smtp.example.com`. Hit "Get new email" and look at the
headers of the emails (send yourself an email if you don't have any!) -
there should be an X-Spambayes-Classification header there. It probably
Index: index.ht
===================================================================
RCS file: /cvsroot/spambayes/website/index.ht,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** index.ht 8 Sep 2003 10:55:58 -0000 1.25
--- index.ht 11 Sep 2003 07:58:34 -0000 1.26
***************
*** 155,160 ****
background page for a link). Rob Hooft
also contributed maths/stats clues.
! Mark Hammond amazed the world with the Outlook2000 plug-in (with Tony Meyer
! Sean True and Adam Walker making significant contributions),
and Richie Hindle, Neale Pickett, Tim Stone worked on the end-user applications.
(Thanks also to Rachel Holkner for turning Anthony's gibberish into something
--- 155,160 ----
background page for a link). Rob Hooft
also contributed maths/stats clues.
! Mark Hammond amazed the world with the Outlook2000 plug-in (with Tony Meyer,
! Sean True, and Adam Walker making significant contributions),
and Richie Hindle, Neale Pickett, Tim Stone worked on the end-user applications.
(Thanks also to Rachel Holkner for turning Anthony's gibberish into something
From sjoerd at users.sourceforge.net Thu Sep 11 02:34:44 2003
From: sjoerd at users.sourceforge.net (Sjoerd Mullender)
Date: Thu Sep 11 04:34:47 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.32,1.33
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30693
Modified Files:
storage.py
Log Message:
Typo in comment.
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** storage.py 10 Sep 2003 21:08:24 -0000 1.32
--- storage.py 11 Sep 2003 08:34:42 -0000 1.33
***************
*** 142,146 ****
def close(self):
! # we keep no reasources open - nothing to do
pass
--- 142,146 ----
def close(self):
! # we keep no resources open - nothing to do
pass
From richie at entrian.com Thu Sep 11 11:14:23 2003
From: richie at entrian.com (Richie Hindle)
Date: Thu Sep 11 05:14:49 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32
Message-ID:
Tony,
> It's a big ask to check that *every* part of the code still works.
I'm not asking that people test every part of the code after making an
edit. I'm asking that people *run* the specific code they've edited -
that should be no kind of "ask", let alone a big one.
The fact that this is one of Skip's edits we're talking about proves that
no amount of expertise with a language is enough to be sure that an edit
works without at least running it. It also proves that when you can't
even find your toothbrush, close attention to detail can go by the wayside
8-) - how's the house move, Skip?
--
Richie Hindle
richie@entrian.com
From skip at pobox.com Thu Sep 11 09:52:05 2003
From: skip at pobox.com (Skip Montanaro)
Date: Thu Sep 11 09:52:20 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32
In-Reply-To: <1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz>
References: <1ED4ECF91CDED24C8D012BCF2B034F1303346327@its-xchg4.massey.ac.nz>
<1ED4ECF91CDED24C8D012BCF2B034F13026F2927@its-xchg4.massey.ac.nz>
Message-ID: <16224.32261.481993.169764@montanaro.dyndns.org>
>> Here's a thought: if people would run their edits before committing
>> them, we'd have fewer bug reports. Just an idea.
Tony> To be fair, I tested that the error message printed out correctly
Tony> when I added it, and Skip probably checked that messages were
Tony> going to stderr when he added those, just not the specific case of
Tony> having no dbm modules available.
Are we talking about the "import sys" which occurred after the "print >>
sys.stderr, ..."? If so, I will excuse myself by stating that I always
import modules at the top of a module unless it's unavoidable (as with the
psycopg and MySQLdb imports in the respective SQLClassifier.load() methods,
so it never even occurred to me to look for "import sys" where Richie found
it.
Skip
From skip at pobox.com Thu Sep 11 10:06:09 2003
From: skip at pobox.com (Skip Montanaro)
Date: Thu Sep 11 10:06:21 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.31,1.32
In-Reply-To:
References:
Message-ID: <16224.33105.483423.435036@montanaro.dyndns.org>
>>>>> "Richie" == Richie Hindle writes:
Richie> Tony,
>> It's a big ask to check that *every* part of the code still works.
Richie> I'm not asking that people test every part of the code after
Richie> making an edit. I'm asking that people *run* the specific code
Richie> they've edited - that should be no kind of "ask", let alone a
Richie> big one.
Yeah, but I run that code all the time. I filter all my mail using
SpamBayes CVS. I just never have no database available, so that particular
print was never executed.
Richie> 8-) - how's the house move, Skip?
We're out of the old house, but the new house is still a shambles, so we're
bunking with my mum-in-law...
Skip
From montanaro at users.sourceforge.net Thu Sep 11 08:22:40 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 11 10:22:45 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.20,1.21
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9101
Modified Files:
setup.py
Log Message:
fix incorrect cmdclass key in setup()
Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** setup.py 11 Sep 2003 07:47:52 -0000 1.20
--- setup.py 11 Sep 2003 14:22:38 -0000 1.21
***************
*** 72,76 ****
author_email = "spambayes@python.org",
url = "http://spambayes.sourceforge.net",
! cmdclass = {'build_py': install_scripts},
scripts=['scripts/sb_client.py',
'scripts/sb_dbexpimp.py',
--- 72,76 ----
author_email = "spambayes@python.org",
url = "http://spambayes.sourceforge.net",
! cmdclass = {'install_scripts': install_scripts},
scripts=['scripts/sb_client.py',
'scripts/sb_dbexpimp.py',
From tim_one at users.sourceforge.net Thu Sep 11 08:53:41 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Thu Sep 11 10:53:50 2003
Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv15539
Modified Files:
NEWTRICKS.txt
Log Message:
Added comment about numeric character entities.
Index: NEWTRICKS.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/NEWTRICKS.txt,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** NEWTRICKS.txt 11 Sep 2003 06:57:42 -0000 1.1
--- NEWTRICKS.txt 11 Sep 2003 14:53:39 -0000 1.2
***************
*** 2,9 ****
results of these ideas.
! - If the email parser was able to emit warnings about misformed MIME
that it encountered, we could turn this warnings into tokens.
! - The numeric entity support looks like it will only support 8-bit
characters - an obvious problem is from wacky wide-char entities.
--- 2,12 ----
results of these ideas.
! - If the email parser was able to emit warnings about misformed MIME
that it encountered, we could turn this warnings into tokens.
! - The numeric entity support looks like it will only support 8-bit
characters - an obvious problem is from wacky wide-char entities.
+ [tim] It replaces those with a question mark, because chr(n) raises
+ an exception for n > 255, and the "except" clause returns '?' then.
+ Is that a problem? Probably not, for Americans and Europeans.
From tim_one at users.sourceforge.net Thu Sep 11 18:22:42 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Thu Sep 11 20:22:45 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.109,1.110
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv10021/Outlook2000
Modified Files:
addin.py
Log Message:
ShowClues(): Add lines revealing the # ham and spam trained on.
Massive imbalance is turning out to be a problem for some people, and it
takes too many steps to guess that now.
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.109
retrieving revision 1.110
diff -C2 -d -r1.109 -r1.110
*** addin.py 8 Sep 2003 00:14:14 -0000 1.109
--- addin.py 12 Sep 2003 00:22:39 -0000 1.110
***************
*** 139,143 ****
import filter
filter.filter_message(msgstore_message, manager, all_actions = False)
!
else:
print "already was trained as good"
--- 139,143 ----
import filter
filter.filter_message(msgstore_message, manager, all_actions = False)
!
else:
print "already was trained as good"
***************
*** 171,175 ****
% msgstore_message.subject)
return
!
if HaveSeenMessage(msgstore_message, manager):
# Already seen this message - user probably moving it back
--- 171,175 ----
% msgstore_message.subject)
return
!
if HaveSeenMessage(msgstore_message, manager):
# Already seen this message - user probably moving it back
***************
*** 191,195 ****
# rules moving the item.
need_train = manager.config.filter.unsure_threshold < prop * 100
!
if need_train:
TrainAsHam(msgstore_message, manager)
--- 191,195 ----
# rules moving the item.
need_train = manager.config.filter.unsure_threshold < prop * 100
!
if need_train:
TrainAsHam(msgstore_message, manager)
***************
*** 455,463 ****
body = ["Spam Score: %d%% (%g)
" % (round(score*100), score)]
push = body.append
! # Format the clues.
push("\n")
push("word spamprob #ham #spam\n")
format = " %-12g %8s %6s\n"
- c = mgr.GetClassifier()
fetchword = c.wordinfo.get
for word, prob in clues:
--- 455,467 ----
body = ["Spam Score: %d%% (%g)
" % (round(score*100), score)]
push = body.append
! # Format the # ham and spam trained on.
! c = mgr.GetClassifier()
push("\n")
+ push("# ham trained on: %6d\n" % c.nham)
+ push("# spam trained on: %6d\n" % c.nspam)
+ push("\n")
+ # Format the clues.
push("word spamprob #ham #spam\n")
format = " %-12g %8s %6s\n"
fetchword = c.wordinfo.get
for word, prob in clues:
***************
*** 671,675 ****
print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
restore_folder = inbox_folder
!
# Must train before moving, else we lose the message!
print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
--- 675,679 ----
print "Unable to determine source folder for message '%s' - restoring to Inbox" % (subject,)
restore_folder = inbox_folder
!
# Must train before moving, else we lose the message!
print "Recovering to folder '%s' and ham training message '%s' - " % (restore_folder.name, subject),
***************
*** 1157,1161 ****
assert self.manager.addin is None, "Should not already have an addin"
self.manager.addin = self
!
# Only now will the import of "spambayes.Version" work, as the
# manager is what munges sys.path for us.
--- 1161,1165 ----
assert self.manager.addin is None, "Should not already have an addin"
self.manager.addin = self
!
# Only now will the import of "spambayes.Version" work, as the
# manager is what munges sys.path for us.
From mhammond at users.sourceforge.net Fri Sep 12 05:58:55 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 12 07:59:02 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv30094
Modified Files:
pop3proxy_tray.py
Log Message:
* Use simple logging strategy similar to the Outlook addin - if we have
no console, redirect either to win32traceutil (source-code version) or
to a %TEMP\SpamBayesServer1.log (yet to be released binary version).
* When running from binary, don't hack sys.path. When running from source
code, hack sys.path based file path rather than on os.getcwd.
* When running from binary, load the icon from the executable rather than
a .ico file.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** pop3proxy_tray.py 9 Sep 2003 01:07:26 -0000 1.6
--- pop3proxy_tray.py 12 Sep 2003 11:58:53 -0000 1.7
***************
*** 29,37 ****
from win32api import *
from win32gui import *
! # Allow for those without SpamBayes on the PYTHONPATH
! sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.dirname(os.getcwd()))
! sys.path.insert(-1, os.path.join(os.path.dirname(os.getcwd()),"scripts"))
import sb_server
--- 29,84 ----
from win32api import *
from win32gui import *
+ from win32api import error as win32api_error
! # If we are not running in a console, redirect all print statements to the
! # win32traceutil collector.
! # You can view output either from Pythonwin's "Tools->Trace Collector Debugging Tool",
! # or simply run "win32traceutil.py" from a command prompt.
! try:
! GetConsoleTitle()
! except win32api_error:
! # No console - if we are running from Python sources,
! # redirect to win32traceutil, but if running from a binary
! # install, redirect to a log file.
! # Want to move to logging module later, so for now, we
! # hack together a simple logging strategy.
! if hasattr(sys, "frozen"):
! temp_dir = GetTempPath()
! for i in range(3,0,-1):
! try: os.unlink(os.path.join(temp_dir, "SpamBayesServer%d.log" % (i+1)))
! except os.error: pass
! try:
! os.rename(
! os.path.join(temp_dir, "SpamBayesServer%d.log" % i),
! os.path.join(temp_dir, "SpamBayesServer%d.log" % (i+1))
! )
! except os.error: pass
! # Open this log, as unbuffered so crashes still get written.
! sys.stdout = open(os.path.join(temp_dir,"SpamBayesServer1.log"), "wt", 0)
! sys.stderr = sys.stdout
! else:
! import win32traceutil
!
! # Work out our "application directory", which is
! # the directory of our main .py/.exe file we
! # are running from.
! try:
! if hasattr(sys, "frozen"):
! if sys.frozen == "dll":
! # Don't think we will ever run as a .DLL, but...
! this_filename = win32api.GetModuleFileName(sys.frozendllhandle)
! else:
! this_filename = os.path.abspath(sys.argv[0])
! else:
! this_filename = os.path.abspath(__file__)
! except NameError: # no __file__
! this_filename = os.path.abspath(sys.argv[0])
!
! this_dir = os.path.dirname(this_filename)
! if not hasattr(sys, "frozen"):
! # Allow for those without SpamBayes on the PYTHONPATH
! sys.path.insert(-1, this_dir)
! sys.path.insert(-1, os.path.dirname(this_dir))
! sys.path.insert(-1, os.path.join(os.path.dirname(this_dir),"scripts"))
import sb_server
***************
*** 47,51 ****
# The ordering here is important - it is the order that they will
# appear in the menu. As dicts don't have an order, this means
! # that the order is controlled by the id. Any items were the
# function is None will appear as separators.
self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
--- 94,98 ----
# The ordering here is important - it is the order that they will
# appear in the menu. As dicts don't have an order, this means
! # that the order is controlled by the id. Any items where the
# function is None will appear as separators.
self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
***************
*** 81,85 ****
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
## (os.path.dirname(sb_server.__file__),)
! if os.path.isfile(startedIconPathName) and os.path.isfile(stoppedIconPathName):
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0,
--- 128,142 ----
## iconPathName = "%s\\..\\windows\\resources\\sbicon.ico" % \
## (os.path.dirname(sb_server.__file__),)
! if hasattr(sys, "frozen"):
! self.hstartedicon = self.hstoppedicon = None
! hexe = GetModuleHandle(None)
! icon_flags = win32con.LR_DEFAULTSIZE
! self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
! self.hstopped = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
! 0, icon_flags)
! else:
! # If we have no icon we fail in all sorts of places - so may as
! # well make it here :)
icon_flags = win32con.LR_LOADFROMFILE | win32con.LR_DEFAULTSIZE
self.hstartedicon = LoadImage(hinst, startedIconPathName, win32con.IMAGE_ICON, 0,
***************
*** 164,167 ****
--- 221,225 ----
except KeyError:
print "Unknown command -", id
+ return
function()
From mhammond at users.sourceforge.net Fri Sep 12 06:00:04 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 12 08:00:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv30773
Modified Files:
setup_all.py
Log Message:
Add pop3proxy_tray as a GUI program. Add the icons to the executables, and
sys.path hacks for the new world order.
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** setup_all.py 29 Aug 2003 14:44:08 -0000 1.1
--- setup_all.py 12 Sep 2003 12:00:02 -0000 1.2
***************
*** 4,21 ****
import sys, os
sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
! try:
! import classifier
! except ImportError:
! sys.path.append(sb_top_dir)
!
! try:
! import pop3proxy_service
! except ImportError:
! sys.path.append(os.path.join(sb_top_dir, "windows"))
!
! try:
! import addin
! except ImportError:
! sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
# ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
--- 4,11 ----
import sys, os
sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
! sys.path.append(sb_top_dir)
! sys.path.append(os.path.join(sb_top_dir, "windows"))
! sys.path.append(os.path.join(sb_top_dir, "scripts"))
! sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
# ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
***************
*** 41,44 ****
--- 31,37 ----
class py2exe_options:
bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))]
+ icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
+ (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico")),
+ ]
packages = "spambayes.resources"
excludes = "win32ui,pywin" # pywin is a package, and still seems to be included.
***************
*** 48,52 ****
if len(sys.argv)==1:
sys.argv = [sys.argv[0], "py2exe"]
!
setup(name="SpamBayes",
packages = ["spambayes.resources"],
--- 41,45 ----
if len(sys.argv)==1:
sys.argv = [sys.argv[0], "py2exe"]
!
setup(name="SpamBayes",
packages = ["spambayes.resources"],
***************
*** 56,59 ****
service=["pop3proxy_service"],
# A console exe for debugging
! console=[os.path.join(sb_top_dir, "pop3proxy.py")],
)
--- 49,54 ----
service=["pop3proxy_service"],
# A console exe for debugging
! console=[os.path.join(sb_top_dir, "scripts", "sb_server.py")],
! # The taskbar
! windows=[os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py")],
)
From anadelonbrin at users.sourceforge.net Sat Sep 13 00:03:43 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 13 00:03:47 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv29764/scripts
Modified Files:
sb_smtpproxy.py
Log Message:
Fix [ 805351 ] If cc: address is not qualified, proxy fails to send message
Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_smtpproxy.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** sb_smtpproxy.py 5 Sep 2003 01:16:45 -0000 1.1
--- sb_smtpproxy.py 13 Sep 2003 04:03:41 -0000 1.2
***************
*** 310,321 ****
return address
- def splitTo(self, address):
- """Return 'address' as undressed (host, fulladdress) tuple.
- Handy for use with TO: addresses."""
- start = string.index(address, '<') + 1
- sep = string.index(address, '@') + 1
- end = string.index(address, '>')
- return (address[sep:end], address[start:end],)
-
def onTransaction(self, command, args):
handler = self.handlers.get(command.upper(), self.onUnknown)
--- 310,313 ----
***************
*** 334,338 ****
def onRcptTo(self, command, args):
! toHost, toFull = self.splitTo(args[0])
if toFull == options["smtpproxy", "spam_address"]:
self.train_as_spam = True
--- 326,330 ----
def onRcptTo(self, command, args):
! toFull = self.stripAddress(args[0])
if toFull == options["smtpproxy", "spam_address"]:
self.train_as_spam = True
From anadelonbrin at users.sourceforge.net Sat Sep 13 01:10:51 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sat Sep 13 01:10:55 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv8944/scripts
Modified Files:
sb_server.py
Log Message:
It's probably a good idea to close and save the database *before* we turn it into a completely
new one...
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** sb_server.py 10 Sep 2003 04:33:17 -0000 1.3
--- sb_server.py 13 Sep 2003 05:10:49 -0000 1.4
***************
*** 714,718 ****
def _recreateState():
global state
- state = State()
# Close the existing listeners and create new ones. This won't
--- 714,717 ----
***************
*** 727,730 ****
--- 726,731 ----
state.bayes.store()
state.bayes.close()
+
+ state = State()
prepare(state)
From mhammond at users.sourceforge.net Sun Sep 14 01:23:53 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 01:24:01 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv28887
Modified Files:
setup_all.py
Log Message:
New the new py2exe sandbox "targets" feature I ject checked in. This now
works for everything bar Outlook, and that is *very* close (no dialogs
yet!)
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** setup_all.py 12 Sep 2003 12:00:02 -0000 1.2
--- setup_all.py 14 Sep 2003 05:23:51 -0000 1.3
***************
*** 17,21 ****
modulefinder.AddPackagePath("win32com", p)
# Not sure why this works for "win32com.mapi" for not "win32com.shell"!
! for extra in ["win32com.shell"]:
__import__(extra)
m = sys.modules[extra]
--- 17,21 ----
modulefinder.AddPackagePath("win32com", p)
# Not sure why this works for "win32com.mapi" for not "win32com.shell"!
! for extra in ["win32com.shell","win32com.mapi"]:
__import__(extra)
m = sys.modules[extra]
***************
*** 29,40 ****
import py2exe
! class py2exe_options:
! bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))]
! icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
! (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico")),
! ]
! packages = "spambayes.resources"
! excludes = "win32ui,pywin" # pywin is a package, and still seems to be included.
# Default and only distutils command is "py2exe" - save adding it to the
# command line every single time.
--- 29,61 ----
import py2exe
! class Options:
! def __init__(self, **kw):
! self.__dict__.update(kw)
+ # py2exe_options is a global name found by py2exe
+ py2exe_options = Options(
+ packages = "spambayes.resources,encodings",
+ excludes = "win32ui,pywin,pywin.debugger" # pywin is a package, and still seems to be included.
+ )
+
+ # These are just objects passed to py2exe
+ com_server = Options(
+ modules = ["addin"],
+ dest_base = "SpamBayes_Outlook_Addin",
+ bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))],
+ create_exe = False,
+ )
+
+ service = Options(
+ modules = ["pop3proxy_service"]
+ )
+ sb_server = Options(
+ script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
+ )
+ pop3proxy_tray = Options(
+ script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
+ icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
+ (1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico"))],
+ )
# Default and only distutils command is "py2exe" - save adding it to the
# command line every single time.
***************
*** 45,54 ****
packages = ["spambayes.resources"],
# We implement a COM object.
! com_server=["addin"],
# A service
! service=["pop3proxy_service"],
# A console exe for debugging
! console=[os.path.join(sb_top_dir, "scripts", "sb_server.py")],
# The taskbar
! windows=[os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py")],
)
--- 66,75 ----
packages = ["spambayes.resources"],
# We implement a COM object.
! com_server=[com_server],
# A service
! service=[service],
# A console exe for debugging
! console=[sb_server],
# The taskbar
! windows=[pop3proxy_tray],
)
From tim_one at users.sourceforge.net Sun Sep 14 16:24:15 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Sun Sep 14 16:24:18 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.110,1.111
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv16908/Outlook2000
Modified Files:
addin.py
Log Message:
ShowClues(): Made the clue report a little prettier, and (I hope) a
little easier to follow.
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.110
retrieving revision 1.111
diff -C2 -d -r1.110 -r1.111
*** addin.py 12 Sep 2003 00:22:39 -0000 1.110
--- addin.py 14 Sep 2003 20:24:12 -0000 1.111
***************
*** 453,466 ****
# way to get this as text only. Next best then is to use HTML, 'cos at
# least we know how to exploit it!
! body = ["Spam Score: %d%% (%g)
" % (round(score*100), score)]
push = body.append
# Format the # ham and spam trained on.
c = mgr.GetClassifier()
! push("\n")
! push("# ham trained on: %6d\n" % c.nham)
! push("# spam trained on: %6d\n" % c.nspam)
! push("\n")
# Format the clues.
! push("word spamprob #ham #spam\n")
format = " %-12g %8s %6s\n"
fetchword = c.wordinfo.get
--- 453,472 ----
# way to get this as text only. Next best then is to use HTML, 'cos at
# least we know how to exploit it!
! body = ["Combined Score: %d%% (%g)
\n" %
! (round(score*100), score)]
push = body.append
+ # Format internal scores.
+ push("Internal ham score (%s): %g
\n" % clues.pop(0))
+ push("Internal spam score (%s): %g
\n" % clues.pop(0))
# Format the # ham and spam trained on.
c = mgr.GetClassifier()
! push("
\n")
! push("# ham trained on: %d
\n" % c.nham)
! push("# spam trained on: %d
\n" % c.nspam)
# Format the clues.
! push("%s Significant Tokens
\n" % len(clues))
! push("")
! push("token spamprob #ham #spam\n")
! push("")
format = " %-12g %8s %6s\n"
fetchword = c.wordinfo.get
***************
*** 478,482 ****
# Now the raw text of the message, as best we can
! push("Message Stream:
")
push("\n")
msg = msgstore_message.GetEmailPackageObject(strip_mime_headers=False)
--- 484,488 ----
# Now the raw text of the message, as best we can
! push("Message Stream
\n")
push("\n")
msg = msgstore_message.GetEmailPackageObject(strip_mime_headers=False)
***************
*** 487,491 ****
from spambayes.tokenizer import tokenize
from spambayes.classifier import Set # whatever classifier uses
! push("Message Tokens:
")
# need to re-fetch, as the tokens we see may be different based on
# header stripping.
--- 493,497 ----
from spambayes.tokenizer import tokenize
from spambayes.classifier import Set # whatever classifier uses
! push("All Message Tokens
\n")
# need to re-fetch, as the tokens we see may be different based on
# header stripping.
***************
*** 506,510 ****
new_msg.Subject = "Spam Clues: " + item.Subject
# As above, use HTMLBody else Outlook refuses to behave.
! new_msg.HTMLBody = "" + body + ""
# Attach the source message to it
# Using the original message has the side-effect of marking the original
--- 512,523 ----
new_msg.Subject = "Spam Clues: " + item.Subject
# As above, use HTMLBody else Outlook refuses to behave.
! new_msg.HTMLBody = """\
!
!
!
!
! """ + body + ""
# Attach the source message to it
# Using the original message has the side-effect of marking the original
From tim_one at users.sourceforge.net Sun Sep 14 19:46:51 2003
From: tim_one at users.sourceforge.net (Tim Peters)
Date: Sun Sep 14 19:46:54 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.20,1.21
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27286/spambayes
Modified Files:
Version.py
Log Message:
SF bug 806238: urllib2 fails in Outlook new-version chk.
Switch to a SF URL until the problem with the spambayes.org URL is
understood.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -C2 -d -r1.20 -r1.21
*** Version.py 9 Sep 2003 03:09:49 -0000 1.20
--- Version.py 14 Sep 2003 23:46:49 -0000 1.21
***************
*** 11,15 ****
"""
! LATEST_VERSION_HOME="http://www.spambayes.org/download/Version.cfg"
# This module is part of the spambayes project, which is Copyright 2002-3
--- 11,19 ----
"""
! # See bug 806238: urllib2 fails in Outlook new-version chk.
! # A reason for why the spambayes.org URL fails is given in a comment there.
! #LATEST_VERSION_HOME="http://www.spambayes.org/download/Version.cfg"
! # The SF URL instead works for Tim and xenogeist.
! LATEST_VERSION_HOME="http://spambayes.sourceforge.net/download/Version.cfg"
# This module is part of the spambayes project, which is Copyright 2002-3
***************
*** 187,191 ****
for appname in versions["Apps"]:
_make_cfg_section(stream, appname, versions["Apps"][appname])
!
def main(args):
if '-g' in args:
--- 191,195 ----
for appname in versions["Apps"]:
_make_cfg_section(stream, appname, versions["Apps"][appname])
!
def main(args):
if '-g' in args:
From anadelonbrin at users.sourceforge.net Sun Sep 14 20:37:52 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 20:37:55 2003
Subject: [Spambayes-checkins] website docs.ht,1.15,1.16
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv5196
Modified Files:
docs.ht
Log Message:
Fix Outlook readme and troubleshooting doc displaying in plain text in some browsers.
Index: docs.ht
===================================================================
RCS file: /cvsroot/spambayes/website/docs.ht,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** docs.ht 1 Jul 2003 00:50:11 -0000 1.15
--- docs.ht 15 Sep 2003 00:37:50 -0000 1.16
***************
*** 10,14 ****
A rudimentary set of Frequently Asked Questions (FAQ).
Instructions on integrating Spambayes into your mail system.
! The Outlook plugin includes an "About" File, and a "Troubleshooting Guide" that can be accessed via the toolbar.
(Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
Always start with the documentation that came with the version you installed.)
--- 10,15 ----
A rudimentary set of Frequently Asked Questions (FAQ).
Instructions on integrating Spambayes into your mail system.
! The Outlook plugin includes an "About" File, and a
! "Troubleshooting Guide" that can be accessed via the toolbar.
(Note that the online documentaton is always for the latest source version, and so might not correspond exactly with the version you are using.
Always start with the documentation that came with the version you installed.)
From anadelonbrin at users.sourceforge.net Sun Sep 14 21:00:20 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:00:23 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.21,1.22
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9175/spambayes
Modified Files:
Version.py
Log Message:
Add a (basic) check for version option to the pop3proxy tray app.
Also add binary version info for pop3proxy. Since pop3proxy and Outlook will be using
the same installer, should this maybe be in some central location, rather than in each
of them?
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** Version.py 14 Sep 2003 23:46:49 -0000 1.21
--- Version.py 15 Sep 2003 01:00:18 -0000 1.22
***************
*** 49,52 ****
--- 49,53 ----
"POP3 Proxy" : {
"Version": 0.2,
+ "BinaryVersion": 0.1,
"Description": "SpamBayes POP3 Proxy Beta2",
"Date": "September 2003",
***************
*** 55,58 ****
--- 56,66 ----
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
+ "Full Description Binary":
+ """%(Description)s, Binary version %(BinaryVersion)s (%(Date)s),
+ using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
+ # Note this means we can change the download page later, and old
+ # versions will still go to the new page.
+ # We may also like to have a "Release Notes Page" item later?
+ "Download Page": "http://spambayes.sourceforge.net/windows.html"
},
"Lotus Notes Filter" : {
From anadelonbrin at users.sourceforge.net Sun Sep 14 21:00:20 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:00:25 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.7,1.8
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv9175/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Add a (basic) check for version option to the pop3proxy tray app.
Also add binary version info for pop3proxy. Since pop3proxy and Outlook will be using
the same installer, should this maybe be in some central location, rather than in each
of them?
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** pop3proxy_tray.py 12 Sep 2003 11:58:53 -0000 1.7
--- pop3proxy_tray.py 15 Sep 2003 01:00:18 -0000 1.8
***************
*** 25,28 ****
--- 25,41 ----
import webbrowser
import thread
+ import traceback
+
+ # This should just be imported from dialogs.dlgutils, but
+ # I'm not sure that we can import from the Outlook2000
+ # directory, because I don't think it gets installed.
+ ##from spambayes.Outlook2000.dialogs.dlgutils import SetWaitCursor
+ def SetWaitCursor(wait):
+ import win32gui, win32con
+ if wait:
+ hCursor = win32gui.LoadCursor(0, win32con.IDC_WAIT)
+ else:
+ hCursor = win32gui.LoadCursor(0, 0)
+ win32gui.SetCursor(hCursor)
import win32con
***************
*** 100,105 ****
1026 : ("View information ...", self.OpenInterface),
1027 : ("Configure ...", self.OpenConfig),
1028 : ("-", None),
! 1029 : ("Exit SpamBayes", self.OnExit),
}
message_map = {
--- 113,119 ----
1026 : ("View information ...", self.OpenInterface),
1027 : ("Configure ...", self.OpenConfig),
+ 1027 : ("Check for latest version", self.CheckVersion),
1028 : ("-", None),
! 1099 : ("Exit SpamBayes", self.OnExit),
}
message_map = {
***************
*** 260,263 ****
--- 274,312 ----
webbrowser.open_new("http://localhost:%d/config" % \
(options["html_ui", "port"],))
+
+ def CheckVersion(self):
+ # Stolen, with few modifications, from addin.py
+ from spambayes.Version import get_version_string, \
+ get_version_number, fetch_latest_dict
+ if hasattr(sys, "frozen"):
+ version_number_key = "BinaryVersion"
+ version_string_key = "Full Description Binary"
+ else:
+ version_number_key = "Version"
+ version_string_key = "Full Description"
+
+ app_name = "POP3 Proxy"
+ cur_ver_string = get_version_string(app_name, version_string_key)
+ cur_ver_num = get_version_number(app_name, version_number_key)
+
+ try:
+ SetWaitCursor(1)
+ latest = fetch_latest_dict()
+ SetWaitCursor(0)
+ latest_ver_string = get_version_string(app_name, version_string_key,
+ version_dict=latest)
+ latest_ver_num = get_version_number(app_name, version_number_key,
+ version_dict=latest)
+ except:
+ print "Error checking the latest version"
+ traceback.print_exc()
+ return
+
+ print "Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num)
+ if latest_ver_num > cur_ver_num:
+ url = get_version_string(app_name, "Download Page", version_dict=latest)
+ # Offer to open up the url
+ ## os.startfile(url)
+
def main():
From anadelonbrin at users.sourceforge.net Sun Sep 14 21:08:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:08:17 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000
default_bayes_customize.ini, 1.7, 1.8
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv10633/Outlook2000
Modified Files:
default_bayes_customize.ini
Log Message:
Goodbye Gary-combining! (Use the Last-Gary tag to get it back).
Index: default_bayes_customize.ini
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/default_bayes_customize.ini,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** default_bayes_customize.ini 18 Nov 2002 01:40:04 -0000 1.7
--- default_bayes_customize.ini 15 Sep 2003 01:08:11 -0000 1.8
***************
*** 4,9 ****
[Tokenizer]
! # So long as Mark, Sean, and I are the primary users of this (i.e.,
! # Americans and Australians), this non-default option is very effective
# at nailing Asian spam with little training and small database burden.
# It should probably be exposed via the GUI, as it's not appropriate
--- 4,8 ----
[Tokenizer]
! # This non-default option is very effective
# at nailing Asian spam with little training and small database burden.
# It should probably be exposed via the GUI, as it's not appropriate
***************
*** 24,32 ****
[Classifier]
- # Uncomment the next lines if you want to use the former default for
- # scoring.
- #use_chi_squared_combining: False
- #use_gary_combining: True
-
# This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True
\ No newline at end of file
--- 23,26 ----
[Classifier]
# This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True
From anadelonbrin at users.sourceforge.net Sun Sep 14 21:08:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Sun Sep 14 21:08:19 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.72,
1.73 classifier.py, 1.9, 1.10
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10633/spambayes
Modified Files:
Options.py classifier.py
Log Message:
Goodbye Gary-combining! (Use the Last-Gary tag to get it back).
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.72
retrieving revision 1.73
diff -C2 -d -r1.72 -r1.73
*** Options.py 9 Sep 2003 07:03:54 -0000 1.72
--- Options.py 15 Sep 2003 01:08:11 -0000 1.73
***************
*** 371,382 ****
REAL, RESTORE),
- ("use_gary_combining", "Use gary-combining", False,
- """The combining scheme currently detailed on the Robinson web page.
- The middle ground here is touchy, varying across corpus, and within
- a corpus across amounts of training data. It almost never gives
- extreme scores (near 0.0 or 1.0), but the tail ends of the ham and
- spam distributions overlap.""",
- BOOLEAN, RESTORE),
-
("use_chi_squared_combining", "Use chi-squared combining", True,
"""For vectors of random, uniformly distributed probabilities,
--- 371,374 ----
Index: classifier.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/classifier.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** classifier.py 5 Sep 2003 01:15:28 -0000 1.9
--- classifier.py 15 Sep 2003 01:08:11 -0000 1.10
***************
*** 25,28 ****
--- 25,31 ----
# advance, and it's touchy.
#
+ # The last version of the Gary-combining scheme can be retrieved from our
+ # CVS repository via tag Last-Gary.
+ #
# The chi-combining scheme used by default here gets closer to the theoretical
# basis of Gary's combining scheme, and does give extreme scores, but also
***************
*** 108,181 ****
# spamprob() implementations. One of the following is aliased to
# spamprob, depending on option settings.
!
! def gary_spamprob(self, wordstream, evidence=False):
! """Return best-guess probability that wordstream is spam.
!
! wordstream is an iterable object producing words.
! The return value is a float in [0.0, 1.0].
!
! If optional arg evidence is True, the return value is a pair
! probability, evidence
! where evidence is a list of (word, probability) pairs.
! """
!
! from math import frexp
!
! # This combination method is due to Gary Robinson; see
! # http://radio.weblogs.com/0101454/stories/2002/09/16/spamDetection.html
!
! # The real P = this P times 2**Pexp. Likewise for Q. We're
! # simulating unbounded dynamic float range by hand. If this pans
! # out, *maybe* we should store logarithms in the database instead
! # and just add them here. But I like keeping raw counts in the
! # database (they're easy to understand, manipulate and combine),
! # and there's no evidence that this simulation is a significant
! # expense.
! P = Q = 1.0
! Pexp = Qexp = 0
! clues = self._getclues(wordstream)
! for prob, word, record in clues:
! P *= 1.0 - prob
! Q *= prob
! if P < 1e-200: # move back into range
! P, e = frexp(P)
! Pexp += e
! if Q < 1e-200: # move back into range
! Q, e = frexp(Q)
! Qexp += e
!
! P, e = frexp(P)
! Pexp += e
! Q, e = frexp(Q)
! Qexp += e
!
! num_clues = len(clues)
! if num_clues:
! #P = 1.0 - P**(1./num_clues)
! #Q = 1.0 - Q**(1./num_clues)
! #
! # (x*2**e)**n = x**n * 2**(e*n)
! n = 1.0 / num_clues
! P = 1.0 - P**n * 2.0**(Pexp * n)
! Q = 1.0 - Q**n * 2.0**(Qexp * n)
!
! # (P-Q)/(P+Q) is in -1 .. 1; scaling into 0 .. 1 gives
! # ((P-Q)/(P+Q)+1)/2 =
! # ((P-Q+P-Q)/(P+Q)/2 =
! # (2*P/(P+Q)/2 =
! # P/(P+Q)
! prob = P/(P+Q)
! else:
! prob = 0.5
!
! if evidence:
! clues = [(w, p) for p, w, r in clues]
! clues.sort(lambda a, b: cmp(a[1], b[1]))
! return prob, clues
! else:
! return prob
!
! if options["Classifier", "use_gary_combining"]:
! spamprob = gary_spamprob
# Across vectors of length n, containing random uniformly-distributed
--- 111,116 ----
# spamprob() implementations. One of the following is aliased to
# spamprob, depending on option settings.
! # Currently only chi-squared is available, but maybe there will be
! # an alternative again someday.
# Across vectors of length n, containing random uniformly-distributed
From xenogeist at users.sourceforge.net Sun Sep 14 21:35:06 2003
From: xenogeist at users.sourceforge.net (Adam Walker)
Date: Sun Sep 14 21:35:15 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.8,1.9
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv16419/windows
Modified Files:
pop3proxy_tray.py
Log Message:
Support for service from Romain GUY.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** pop3proxy_tray.py 15 Sep 2003 01:00:18 -0000 1.8
--- pop3proxy_tray.py 15 Sep 2003 01:35:04 -0000 1.9
***************
*** 43,46 ****
--- 43,47 ----
from win32gui import *
from win32api import error as win32api_error
+ from win32service import *
# If we are not running in a console, redirect all print statements to the
***************
*** 102,105 ****
--- 103,109 ----
START_STOP_ID = 1024
+ runningStatus = (SERVICE_START_PENDING, SERVICE_RUNNING, SERVICE_CONTINUE_PENDING)
+ stoppedStatus = (SERVICE_PAUSED, SERVICE_STOP_PENDING, SERVICE_STOPPED)
+ serviceName = "pop3proxy"
class MainWindow(object):
***************
*** 113,118 ****
1026 : ("View information ...", self.OpenInterface),
1027 : ("Configure ...", self.OpenConfig),
! 1027 : ("Check for latest version", self.CheckVersion),
! 1028 : ("-", None),
1099 : ("Exit SpamBayes", self.OnExit),
}
--- 117,122 ----
1026 : ("View information ...", self.OpenInterface),
1027 : ("Configure ...", self.OpenConfig),
! 1028 : ("Check for latest version", self.CheckVersion),
! 1029 : ("-", None),
1099 : ("Exit SpamBayes", self.OnExit),
}
***************
*** 122,125 ****
--- 126,131 ----
WM_TASKBAR_NOTIFY : self.OnTaskbarNotify,
}
+ self.use_service = True
+ #self.use_service = False
# Create the Window.
***************
*** 160,167 ****
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon, "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
self.started = False
self.tip = None
# Start up sb_server
--- 166,187 ----
flags = NIF_ICON | NIF_MESSAGE | NIF_TIP
! nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, self.hstartedicon,
! "SpamBayes")
Shell_NotifyIcon(NIM_ADD, nid)
self.started = False
self.tip = None
+
+ try:
+ if self.use_service and self.IsServiceAvailable():
+ if self.GetServiceStatus() in runningStatus:
+ self.started = True
+ else:
+ print "Service not availible. Using thread."
+ self.use_service = False
+ self.started = False
+ except:
+ print "Usage of service failed. Reverting to thread."
+ self.use_service = False
+ self.started = False
# Start up sb_server
***************
*** 169,182 ****
# XXX This should determine if we are using the service, and if so
# XXX start that, and if not kick sb_server off in a separate thread.
! sb_server.prepare(state=sb_server.state)
! self.StartStop()
def BuildToolTip(self):
tip = None
! if self.started == True:
! #%i spam %i unsure %i session %i active
! tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
! (sb_server.state.numSpams, sb_server.state.numHams, sb_server.state.numUnsure,
! sb_server.state.totalSessions, sb_server.state.activeSessions)
else:
tip = "SpamBayes is not running"
--- 189,207 ----
# XXX This should determine if we are using the service, and if so
# XXX start that, and if not kick sb_server off in a separate thread.
! if not self.use_service:
! sb_server.prepare(state=sb_server.state)
! if not self.started:
! self.StartStop()
def BuildToolTip(self):
tip = None
! if self.started:
! if self.use_service:
! tip = "SpamBayes running."
! else:
! tip = "SpamBayes %i spam %i ham %i unsure %i sessions %i active" %\
! (sb_server.state.numSpams, sb_server.state.numHams,
! sb_server.state.numUnsure, sb_server.state.totalSessions,
! sb_server.state.activeSessions)
else:
tip = "SpamBayes is not running"
***************
*** 184,197 ****
! def UpdateIcon(self, hicon=None):
! flags = NIF_TIP
! if hicon is not None:
! flags |= NIF_ICON
else:
! hicon = 0
self.tip = self.BuildToolTip()
nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
Shell_NotifyIcon(NIM_MODIFY, nid)
def OnDestroy(self, hwnd, msg, wparam, lparam):
nid = (self.hwnd, 0)
--- 209,315 ----
! def UpdateIcon(self):
! flags = NIF_TIP | NIF_ICON
! if self.started:
! hicon = self.hstartedicon
else:
! hicon = self.hstoppedicon
self.tip = self.BuildToolTip()
nid = (self.hwnd, 0, flags, WM_TASKBAR_NOTIFY, hicon, self.tip)
Shell_NotifyIcon(NIM_MODIFY, nid)
+ def IsServiceAvailable(self):
+ schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+ schService = OpenService(schSCManager, serviceName,
+ SERVICE_QUERY_STATUS)
+ if schService:
+ CloseServiceHandle(schService)
+ return schService != None
+
+ def GetServiceStatus(self):
+ schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+ schService = OpenService(schSCManager, serviceName,
+ SERVICE_QUERY_STATUS)
+ ssStatus = QueryServiceStatus(schService)
+ CloseServiceHandle(schService)
+ return ssStatus[1]
+
+ def StartService(self):
+ schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+ schService = OpenService(schSCManager, serviceName, SERVICE_START |
+ SERVICE_QUERY_STATUS)
+ # we assume IsServiceAvailable() was called before
+ ssStatus = QueryServiceStatus(schService)
+ if ssStatus[1] in runningStatus:
+ self.started = True
+ CloseServiceHandle(schService)
+ return
+
+ StartService(schService, None)
+
+ ssStatus = QueryServiceStatus(schService)
+ dwStartTickCount = GetTickCount()
+ dwOldCheckPoint = ssStatus[5]
+
+ while ssStatus[1] == SERVICE_START_PENDING:
+ dwWaitTime = ssStatus[6] / 10;
+
+ if dwWaitTime < 1000:
+ dwWaitTime = 1000
+ elif dwWaitTime > 10000:
+ dwWaitTime = 10000
+
+ Sleep(dwWaitTime);
+ ssStatus = QueryServiceStatus(schService)
+
+ if ssStatus[5] > dwOldCheckPoint:
+ dwStartTickCount = GetTickCount()
+ dwOldCheckPoint = ssStatus[5]
+ else:
+ if GetTickCount() - dwStartTickCount > ssStatus[6]:
+ break
+
+ self.started = ssStatus[1] == SERVICE_RUNNING
+ CloseServiceHandle(schService)
+ self.started = True
+
+ def StopService(self):
+ schSCManager = OpenSCManager(None, None, SC_MANAGER_CONNECT)
+ schService = OpenService(schSCManager, serviceName, SERVICE_STOP |
+ SERVICE_QUERY_STATUS)
+ # we assume IsServiceAvailable() was called before
+ ssStatus = QueryServiceStatus(schService)
+ if ssStatus[1] in stoppedStatus:
+ self.started = False
+ CloseServiceHandle(schService)
+ return
+
+ ControlService(schService, SERVICE_CONTROL_STOP)
+
+ ssStatus = QueryServiceStatus(schService)
+ dwStartTickCount = GetTickCount()
+ dwOldCheckPoint = ssStatus[5]
+
+ while ssStatus[1] == SERVICE_STOP_PENDING:
+ dwWaitTime = ssStatus[6] / 10;
+
+ if dwWaitTime < 1000:
+ dwWaitTime = 1000
+ elif dwWaitTime > 10000:
+ dwWaitTime = 10000
+
+ Sleep(dwWaitTime);
+ ssStatus = QueryServiceStatus(schService)
+
+ if ssStatus[5] > dwOldCheckPoint:
+ dwStartTickCount = GetTickCount()
+ dwOldCheckPoint = ssStatus[5]
+ else:
+ if GetTickCount() - dwStartTickCount > ssStatus[6]:
+ break
+
+ CloseServiceHandle(schService)
+ self.started = False
+
def OnDestroy(self, hwnd, msg, wparam, lparam):
nid = (self.hwnd, 0)
***************
*** 239,243 ****
def OnExit(self):
! if self.started:
sb_server.stop(sb_server.state)
self.started = False
--- 357,361 ----
def OnExit(self):
! if self.started and not self.use_service:
sb_server.stop(sb_server.state)
self.started = False
***************
*** 255,278 ****
# XXX start/stop that, and if not kick sb_server off in a separate
# XXX thread, or stop the thread that was started.
! if self.started:
! sb_server.stop(sb_server.state)
! self.started = False
! self.control_functions[START_STOP_ID] = ("Start SpamBayes",
! self.StartStop)
! self.UpdateIcon(self.hstoppedicon)
else:
! self.StartProxyThread()
self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
self.StartStop)
! self.UpdateIcon(self.hstartedicon)
def OpenInterface(self):
! webbrowser.open_new("http://localhost:%d/" % \
! (options["html_ui", "port"],))
def OpenConfig(self):
! webbrowser.open_new("http://localhost:%d/config" % \
! (options["html_ui", "port"],))
!
def CheckVersion(self):
# Stolen, with few modifications, from addin.py
--- 373,411 ----
# XXX start/stop that, and if not kick sb_server off in a separate
# XXX thread, or stop the thread that was started.
! if self.use_service:
! if self.GetServiceStatus() in stoppedStatus:
! self.StartService()
! else:
! self.StopService()
else:
! if self.started:
! sb_server.stop(sb_server.state)
! self.started = False
! else:
! self.StartProxyThread()
! self.UpdateIcon()
! if self.started:
self.control_functions[START_STOP_ID] = ("Stop SpamBayes",
self.StartStop)
! else:
! self.control_functions[START_STOP_ID] = ("Start SpamBayes",
! self.StartStop)
!
!
def OpenInterface(self):
! if self.started:
! webbrowser.open_new("http://localhost:%d/" % \
! (options["html_ui", "port"],))
! else:
! self.ShowMessage("SpamBayes is not running.")
def OpenConfig(self):
! if self.started:
! webbrowser.open_new("http://localhost:%d/config" % \
! (options["html_ui", "port"],))
! else:
! self.ShowMessage("SpamBayes is not running.")
!
def CheckVersion(self):
# Stolen, with few modifications, from addin.py
***************
*** 299,307 ****
version_dict=latest)
except:
! print "Error checking the latest version"
traceback.print_exc()
return
! print "Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num)
if latest_ver_num > cur_ver_num:
url = get_version_string(app_name, "Download Page", version_dict=latest)
--- 432,440 ----
version_dict=latest)
except:
! self.ShowMessage("Error checking the latest version")
traceback.print_exc()
return
! self.ShowMessage("Current version is %s, latest is %s." % (cur_ver_num, latest_ver_num))
if latest_ver_num > cur_ver_num:
url = get_version_string(app_name, "Download Page", version_dict=latest)
***************
*** 309,312 ****
--- 442,448 ----
## os.startfile(url)
+ def ShowMessage(self, msg):
+ MessageBox(self.hwnd, msg, "SpamBayes", win32con.MB_OK)
+
def main():
From mhammond at users.sourceforge.net Sun Sep 14 22:10:21 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 22:10:25 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.3,1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv22215
Modified Files:
setup_all.py
Log Message:
More tweaks to the binary build - correct resources in the addin,
rename the outlook DLL, etc.
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** setup_all.py 14 Sep 2003 05:23:51 -0000 1.3
--- setup_all.py 15 Sep 2003 02:10:19 -0000 1.4
***************
*** 2,6 ****
# A distutils setup script for SpamBayes binaries
! import sys, os
sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
sys.path.append(sb_top_dir)
--- 2,6 ----
# A distutils setup script for SpamBayes binaries
! import sys, os, glob
sb_top_dir = os.path.abspath(os.path.dirname(os.path.join(__file__, "../../../..")))
sys.path.append(sb_top_dir)
***************
*** 8,11 ****
--- 8,12 ----
sys.path.append(os.path.join(sb_top_dir, "scripts"))
sys.path.append(os.path.join(sb_top_dir, "Outlook2000"))
+ sys.path.append(os.path.join(sb_top_dir, "Outlook2000/sandbox"))
# ModuleFinder can't handle runtime changes to __path__, but win32com uses them,
***************
*** 16,20 ****
for p in win32com.__path__[1:]:
modulefinder.AddPackagePath("win32com", p)
- # Not sure why this works for "win32com.mapi" for not "win32com.shell"!
for extra in ["win32com.shell","win32com.mapi"]:
__import__(extra)
--- 17,20 ----
***************
*** 36,49 ****
py2exe_options = Options(
packages = "spambayes.resources,encodings",
! excludes = "win32ui,pywin,pywin.debugger" # pywin is a package, and still seems to be included.
)
# These are just objects passed to py2exe
! com_server = Options(
modules = ["addin"],
! dest_base = "SpamBayes_Outlook_Addin",
! bitmap_resources = [(1000, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp"))],
create_exe = False,
)
service = Options(
--- 36,67 ----
py2exe_options = Options(
packages = "spambayes.resources,encodings",
! excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
! includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
! dll_excludes = ["dapi.dll", "mapi32.dll"]
)
+ # These must be the same IDs as in the dialogs. We really should just extract
+ # them from our rc scripts.
+ outlook_bmp_resources = [
+ ( 125, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sbwizlogo.bmp")),
+ ( 127, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\folders.bmp")),
+ (1062, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp")),
+ ]
+
# These are just objects passed to py2exe
! outlook_addin = Options(
modules = ["addin"],
! dest_base = "spambayes_addin",
! bitmap_resources = outlook_bmp_resources,
create_exe = False,
)
+ #outlook_manager = Options(
+ # script = os.path.join(sb_top_dir, r"Outlook2000\manager.py"),
+ # bitmap_resources = outlook_bmp_resources,
+ #)
+ outlook_dump_props = Options(
+ script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
+ dest_base = "outlook_dump_props",
+ )
service = Options(
***************
*** 58,61 ****
--- 76,80 ----
(1010, os.path.join(sb_top_dir, r"windows\resources\sb-stopped.ico"))],
)
+
# Default and only distutils command is "py2exe" - save adding it to the
# command line every single time.
***************
*** 66,74 ****
packages = ["spambayes.resources"],
# We implement a COM object.
! com_server=[com_server],
# A service
service=[service],
# A console exe for debugging
! console=[sb_server],
# The taskbar
windows=[pop3proxy_tray],
--- 85,93 ----
packages = ["spambayes.resources"],
# We implement a COM object.
! com_server=[outlook_addin],
# A service
service=[service],
# A console exe for debugging
! console=[sb_server, outlook_dump_props],
# The taskbar
windows=[pop3proxy_tray],
From mhammond at users.sourceforge.net Sun Sep 14 22:15:47 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Sun Sep 14 22:15:50 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.9,1.10
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv23230
Modified Files:
pop3proxy_tray.py
Log Message:
Correct icon loading code in the binary.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** pop3proxy_tray.py 15 Sep 2003 01:35:04 -0000 1.9
--- pop3proxy_tray.py 15 Sep 2003 02:15:45 -0000 1.10
***************
*** 154,158 ****
self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
0, icon_flags)
! self.hstopped = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
0, icon_flags)
else:
--- 154,158 ----
self.hstartedicon = LoadImage(hexe, 1000, win32con.IMAGE_ICON, 0,
0, icon_flags)
! self.hstoppedicon = LoadImage(hexe, 1010, win32con.IMAGE_ICON, 0,
0, icon_flags)
else:
From mhammond at users.sourceforge.net Mon Sep 15 00:48:52 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 00:48:56 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.10,1.11
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv15225
Modified Files:
pop3proxy_tray.py
Log Message:
If there was an error shutting down the proxy, just log it and continue
to exit. The error I saw was when it got confused and wasn't even
running, so got a "connection refused" error.
Don't terminate the GUI process via sys.exit - use PostQuitMessage instead.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** pop3proxy_tray.py 15 Sep 2003 02:15:45 -0000 1.10
--- pop3proxy_tray.py 15 Sep 2003 04:48:50 -0000 1.11
***************
*** 358,365 ****
def OnExit(self):
if self.started and not self.use_service:
! sb_server.stop(sb_server.state)
self.started = False
DestroyWindow(self.hwnd)
! sys.exit()
def StartProxyThread(self):
--- 358,371 ----
def OnExit(self):
if self.started and not self.use_service:
! try:
! sb_server.stop(sb_server.state)
! except:
! print "Error stopping proxy at shutdown"
! traceback.print_exc()
! print "Shutting down anyway..."
!
self.started = False
DestroyWindow(self.hwnd)
! PostQuitMessage(0)
def StartProxyThread(self):
From mhammond at users.sourceforge.net Mon Sep 15 01:37:04 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 01:37:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv23882
Modified Files:
setup_all.py
Log Message:
Split the apps into 2 directories, plus a 'lib' directory.
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** setup_all.py 15 Sep 2003 02:10:19 -0000 1.4
--- setup_all.py 15 Sep 2003 05:37:02 -0000 1.5
***************
*** 38,42 ****
excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
! dll_excludes = ["dapi.dll", "mapi32.dll"]
)
--- 38,43 ----
excludes = "win32ui,pywin,pywin.debugger", # pywin is a package, and still seems to be included.
includes = "dialogs.resources.dialogs", # Outlook dynamic dialogs
! dll_excludes = ["dapi.dll", "mapi32.dll"],
! lib_dir = "lib",
)
***************
*** 47,50 ****
--- 48,54 ----
( 127, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\folders.bmp")),
(1062, os.path.join(sb_top_dir, r"Outlook2000\dialogs\resources\sblogo.bmp")),
+ # and these are currently hard-coded in addin.py
+ (6000, os.path.join(sb_top_dir, r"Outlook2000\images\recover_ham.bmp")),
+ (6001, os.path.join(sb_top_dir, r"Outlook2000\images\delete_as_spam.bmp")),
]
***************
*** 52,56 ****
outlook_addin = Options(
modules = ["addin"],
! dest_base = "spambayes_addin",
bitmap_resources = outlook_bmp_resources,
create_exe = False,
--- 56,60 ----
outlook_addin = Options(
modules = ["addin"],
! dest_base = "outlook/spambayes_addin",
bitmap_resources = outlook_bmp_resources,
create_exe = False,
***************
*** 62,75 ****
outlook_dump_props = Options(
script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
! dest_base = "outlook_dump_props",
)
service = Options(
modules = ["pop3proxy_service"]
)
sb_server = Options(
script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
)
pop3proxy_tray = Options(
script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
--- 66,82 ----
outlook_dump_props = Options(
script = os.path.join(sb_top_dir, r"Outlook2000\sandbox\dump_props.py"),
! dest_base = "outlook/outlook_dump_props",
)
service = Options(
+ dest_base = "proxy/pop3proxy_service",
modules = ["pop3proxy_service"]
)
sb_server = Options(
+ dest_base = "proxy/sb_server",
script = os.path.join(sb_top_dir, "scripts", "sb_server.py")
)
pop3proxy_tray = Options(
+ dest_base = "proxy/pop3proxy_tray",
script = os.path.join(sb_top_dir, "windows", "pop3proxy_tray.py"),
icon_resources = [(1000, os.path.join(sb_top_dir, r"windows\resources\sb-started.ico")),
***************
*** 77,80 ****
--- 84,93 ----
)
+ outlook_doc_files = [
+ ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
+ ["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
+ ["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
+ ]
+
# Default and only distutils command is "py2exe" - save adding it to the
# command line every single time.
***************
*** 92,94 ****
--- 105,109 ----
# The taskbar
windows=[pop3proxy_tray],
+ # and the misc data files
+ data_files = outlook_doc_files,
)
From mhammond at users.sourceforge.net Mon Sep 15 02:25:35 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:25:40 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 msgstore.py,1.75,1.76
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv32602
Modified Files:
msgstore.py
Log Message:
* Handle exceptions getting the "receive folders".
* Add a __hash__ method that does the right thing for message objects.
Index: msgstore.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/msgstore.py,v
retrieving revision 1.75
retrieving revision 1.76
diff -C2 -d -r1.75 -r1.76
*** msgstore.py 7 Sep 2003 23:43:21 -0000 1.75
--- msgstore.py 15 Sep 2003 06:25:33 -0000 1.76
***************
*** 370,373 ****
--- 370,378 ----
hex_folder_eid = mapi.HexFromBin(folder_eid)
hex_store_eid = mapi.HexFromBin(store_eid)
+ except pythoncom.com_error, details:
+ if not IsNotAvailableCOMException(details):
+ print "ERROR enumerating a receive folder -", details
+ continue
+ try:
folder = self.GetFolder((hex_store_eid, hex_folder_eid))
# For 'unconfigured' stores, or "stand-alone" PST files,
***************
*** 376,383 ****
if folder.GetParent() is not None:
yield folder
! except pythoncom.com_error, details:
! if not IsNotAvailableCOMException(details):
! print "ERROR enumerating a receive folder -", details
# but we just continue
_MapiTypeMap = {
--- 381,388 ----
if folder.GetParent() is not None:
yield folder
! except MsgStoreException, details:
! print "ERROR opening receive folder -", details
# but we just continue
+ continue
_MapiTypeMap = {
***************
*** 687,695 ****
id_str)
def __eq__(self, other):
- if other is None: return False
ceid = self.msgstore.session.CompareEntryIDs
! return ceid(self.id[0], other.id[0]) and \
! ceid(self.id[1], other.id[1])
def __ne__(self, other):
--- 692,707 ----
id_str)
+ # as per search-key comments above, we also "enforce" this at the Python
+ # level. 2 different messages, but one copied from the other, will
+ # return "==".
+ # Not being consistent could cause subtle bugs, especially in interactions
+ # with various test tools.
+ # Compare the GetID() results if you need to know different messages.
+ def __hash__(self):
+ return hash(self.searchkey)
+
def __eq__(self, other):
ceid = self.msgstore.session.CompareEntryIDs
! return ceid(self.searchkey, other.searchkey)
def __ne__(self, other):
From mhammond at users.sourceforge.net Mon Sep 15 02:26:38 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:26:41 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 addin.py,1.111,1.112
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv321
Modified Files:
addin.py
Log Message:
Support for py2exe - add a "DllRegisterServer" method, and load button
bitmaps from the DLL rather than the source .bmp in binary builds.
Index: addin.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/addin.py,v
retrieving revision 1.111
retrieving revision 1.112
diff -C2 -d -r1.111 -r1.112
*** addin.py 14 Sep 2003 20:24:12 -0000 1.111
--- addin.py 15 Sep 2003 06:26:35 -0000 1.112
***************
*** 1,4 ****
# SpamBayes Outlook Addin
-
import sys, os
import warnings
--- 1,3 ----
***************
*** 721,732 ****
# Also note that the clipboard takes ownership of the handle -
# thus, we can not simply perform this load once and reuse the image.
! if not os.path.isabs(fname):
! # images relative to the application path
! fname = os.path.join(manager.application_directory,
! "images", fname)
! if not os.path.isfile(fname):
! print "WARNING - Trying to use image '%s', but it doesn't exist" % (fname,)
! return None
! handle = win32gui.LoadImage(0, fname, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE | win32con.LR_LOADFROMFILE)
win32clipboard.OpenClipboard()
win32clipboard.SetClipboardData(win32con.CF_BITMAP, handle)
--- 720,741 ----
# Also note that the clipboard takes ownership of the handle -
# thus, we can not simply perform this load once and reuse the image.
! # Hacks for the binary - we can get the bitmaps from resources.
! if hasattr(sys, "frozen"):
! if fname=="recover_ham.bmp":
! bid = 6000
! elif fname=="delete_as_spam.bmp":
! bid = 6001
! else:
! raise RuntimeError, "What bitmap to use for '%s'?" % fname
! handle = win32gui.LoadImage(sys.frozendllhandle, bid, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE)
! else:
! if not os.path.isabs(fname):
! # images relative to the application path
! fname = os.path.join(manager.application_directory,
! "images", fname)
! if not os.path.isfile(fname):
! print "WARNING - Trying to use image '%s', but it doesn't exist" % (fname,)
! return None
! handle = win32gui.LoadImage(0, fname, win32con.IMAGE_BITMAP, 0, 0, win32con.LR_DEFAULTSIZE | win32con.LR_LOADFROMFILE)
win32clipboard.OpenClipboard()
win32clipboard.SetClipboardData(win32con.CF_BITMAP, handle)
***************
*** 1407,1415 ****
pass
if __name__ == '__main__':
import win32com.server.register
win32com.server.register.UseCommandLine(OutlookAddin)
if "--unregister" in sys.argv:
! UnregisterAddin(OutlookAddin)
else:
! RegisterAddin(OutlookAddin)
--- 1416,1430 ----
pass
+ def DllRegisterServer():
+ RegisterAddin(OutlookAddin)
+
+ def DllUnregisterServer():
+ UnregisterAddin(OutlookAddin)
+
if __name__ == '__main__':
import win32com.server.register
win32com.server.register.UseCommandLine(OutlookAddin)
if "--unregister" in sys.argv:
! DllUnregisterServer()
else:
! DllRegisterServer()
From mhammond at users.sourceforge.net Mon Sep 15 02:27:11 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Mon Sep 15 02:27:14 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
__init__.py, 1.3, 1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv461/resources
Modified Files:
__init__.py
Log Message:
Load dialog bitmaps directly from the executable in binary builds.
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/__init__.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** __init__.py 5 Sep 2003 06:53:24 -0000 1.3
--- __init__.py 15 Sep 2003 06:27:09 -0000 1.4
***************
*** 6,9 ****
--- 6,10 ----
if type(bmpid)==type(0):
bmpid = rc_parser.names[bmpid]
+ int_bmpid = rc_parser.ids[bmpid]
# For both binary and source versions, we currently load from files.
# In future py2exe built binary versions we will be able to load the
***************
*** 11,22 ****
filename = rc_parser.bitmaps[bmpid]
if hasattr(sys, "frozen"):
! # bitmap in the app/images directory
! # dont have manager available :(
! dll_filename = win32api.GetModuleFileName(sys.frozendllhandle)
! app_dir = os.path.dirname(dll_filename)
! filename = os.path.join(app_dir, "images", filename)
else:
if not os.path.isabs(filename):
# In this directory
filename = os.path.join( os.path.dirname( __file__ ), filename)
! return 0, filename, win32con.LR_LOADFROMFILE
--- 12,26 ----
filename = rc_parser.bitmaps[bmpid]
if hasattr(sys, "frozen"):
! # in our .exe/.dll - load from that.
! if sys.frozen=="dll":
! hmod = sys.frozendllhandle
! else:
! hmod = win32api.GetModuleHandle(None)
! return hmod, int_bmpid, 0
else:
+ # source code - load the .bmp directly.
if not os.path.isabs(filename):
# In this directory
filename = os.path.join( os.path.dirname( __file__ ), filename)
! return 0, filename, win32con.LR_LOADFROMFILE
! assert 0, "not reached"
From kennypitt at hotmail.com Mon Sep 15 10:22:29 2003
From: kennypitt at hotmail.com (Kenny Pitt)
Date: Mon Sep 15 10:22:52 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.72,
1.73 classifier.py, 1.9, 1.10
In-Reply-To:
Message-ID: <000001c37b94$cd683240$300a10ac@spidynamics.com>
Tony Meyer wrote:
> Update of /cvsroot/spambayes/spambayes/spambayes
> In directory sc8-pr-cvs1:/tmp/cvs-serv10633/spambayes
>
> Modified Files:
> Options.py classifier.py
> Log Message:
> Goodbye Gary-combining! (Use the Last-Gary tag to get it back).
>
Should we also drop the 'use_chi_squared_combining" option? Currently,
setting 'use_chi_squared_combining' to false leaves you with no
Classifier.spamprob() implementation.
--
Kenny Pitt
From anadelonbrin at users.sourceforge.net Mon Sep 15 18:41:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Mon Sep 15 18:41:38 2003
Subject: [Spambayes-checkins] spambayes NEWTRICKS.txt,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv25878
Modified Files:
NEWTRICKS.txt
Log Message:
Ideas that it would be nice to try at some point.
Index: NEWTRICKS.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/NEWTRICKS.txt,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** NEWTRICKS.txt 11 Sep 2003 14:53:39 -0000 1.2
--- NEWTRICKS.txt 15 Sep 2003 22:41:31 -0000 1.3
***************
*** 11,12 ****
--- 11,21 ----
an exception for n > 255, and the "except" clause returns '?' then.
Is that a problem? Probably not, for Americans and Europeans.
+
+ - The ratio of upper-case to lower-case characters in an entire message.
+ (I'm not sure how expensive it would be to calculate this). A lot
+ of the Nigerian scam spams I get are predominately upper-case, and
+ none of my ham.
+
+ - A token indicating the length of a message (rounded to an appropriate
+ level. Also a token indicating the ratio of message length to the
+ number of tokens, and a token indicating the number of tokens.
From anadelonbrin at users.sourceforge.net Tue Sep 16 00:42:34 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 00:46:11 2003
Subject: [Spambayes-checkins] spambayes/spambayes FileCorpus.py,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv25187/spambayes
Modified Files:
FileCorpus.py
Log Message:
Fix [ 795145 ] pop3proxy review page dies with mixed gzip/non messages
Index: FileCorpus.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/FileCorpus.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** FileCorpus.py 5 May 2003 01:22:11 -0000 1.5
--- FileCorpus.py 16 Sep 2003 04:42:32 -0000 1.6
***************
*** 207,223 ****
def load(self):
'''Read the Message substance from the file'''
!
if options["globals", "verbose"]:
print 'loading', self.file_name
pn = self.pathname()
try:
! fp = open(pn, 'rb')
except IOError, e:
if e.errno != errno.ENOENT:
raise
else:
! self.setSubstance(fp.read())
! fp.close()
def store(self):
--- 207,246 ----
def load(self):
'''Read the Message substance from the file'''
! # This is a tricky one! Some people might have a combination
! # of gzip and non-gzip messages, especially when they first
! # change to or from gzip. They should be able to see (but
! # not create) either type, so a FileMessage load needs to be
! # able to load gzip messages, even though it is a FileMessage
! # subclass (GzipFileMessage) that adds the ability to store
! # messages gzipped. If someone can think of a classier (pun
! # intended) way of doing this, be my guest.
if options["globals", "verbose"]:
print 'loading', self.file_name
pn = self.pathname()
+
try:
! fp = gzip.open(pn, 'rb')
except IOError, e:
if e.errno != errno.ENOENT:
raise
else:
! try:
! self.setSubstance(fp.read())
! except IOError, e:
! if str(e) == 'Not a gzipped file':
! # We've probably got both gzipped messages and
! # non-gzipped messages, and need to work with both.
! fp.close()
! try:
! fp = open(self.pathname(), 'rb')
! except IOError, e:
! if e.errno != errno.ENOENT:
! raise
! else:
! self.setSubstance(fp.read())
! fp.close()
! else:
! fp.close()
def store(self):
***************
*** 299,321 ****
class GzipFileMessage(FileMessage):
'''Message that persists as a zipped file system artifact.'''
-
- def load(self):
- '''Read the Message substance from the file'''
-
- if options["globals", "verbose"]:
- print 'loading', self.file_name
-
- pn = self.pathname()
-
- try:
- fp = gzip.open(pn, 'rb')
- except IOError, e:
- if e.errno != errno.ENOENT:
- raise
- else:
- self.setSubstance(fp.read())
- fp.close()
-
-
def store(self):
'''Write the Message substance to the file'''
--- 322,325 ----
From mhammond at users.sourceforge.net Tue Sep 16 01:21:05 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 01:21:08 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.5,1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv31892
Modified Files:
setup_all.py
Log Message:
Add typelibs.
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** setup_all.py 15 Sep 2003 05:37:02 -0000 1.5
--- setup_all.py 16 Sep 2003 05:21:03 -0000 1.6
***************
*** 40,43 ****
--- 40,48 ----
dll_excludes = ["dapi.dll", "mapi32.dll"],
lib_dir = "lib",
+ typelibs = [
+ ('{00062FFF-0000-0000-C000-000000000046}', 0, 9, 0),
+ ('{2DF8D04C-5BFA-101B-BDE5-00AA0044DE52}', 0, 2, 1),
+ ('{AC0714F2-3D04-11D1-AE7D-00A0C90F26F4}', 0, 1, 0),
+ ]
)
***************
*** 101,105 ****
# A service
service=[service],
! # A console exe for debugging
console=[sb_server, outlook_dump_props],
# The taskbar
--- 106,110 ----
# A service
service=[service],
! # console exes for debugging
console=[sb_server, outlook_dump_props],
# The taskbar
From mhammond at users.sourceforge.net Tue Sep 16 10:00:16 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:00:20 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.73,1.74
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv375
Modified Files:
Options.py
Log Message:
If we are running Windows, have no valid config file specified, and have
the win32all extensions available, default to:
\Documents and Settings\[user]\Application Data\SpamBayes\Proxy
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.73
retrieving revision 1.74
diff -C2 -d -r1.73 -r1.74
*** Options.py 15 Sep 2003 01:08:11 -0000 1.73
--- Options.py 16 Sep 2003 14:00:12 -0000 1.74
***************
*** 18,22 ****
"""
! import os
__all__ = ['options']
--- 18,22 ----
"""
! import sys, os
__all__ = ['options']
***************
*** 976,977 ****
--- 976,995 ----
if not optionsPathname:
optionsPathname = os.path.abspath('bayescustomize.ini')
+ if sys.platform.startswith("win") and not os.path.isfile(optionsPathname):
+ # If we are on Windows and still don't have an INI, default to the
+ # 'per-user' directory.
+ try:
+ from win32com.shell import shell, shellcon
+ windowsUserDirectory = os.path.join(
+ shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
+ "SpamBayes", "Proxy")
+ optionsPathname = os.path.join(windowsUserDirectory, 'bayescustomize.ini')
+ except ImportError:
+ # We are on Windows, with no BAYESCUSTOMIZE set, no ini file
+ # in the current directory, and no win32 extensions installed
+ # to locate the "user" directory - seeing things are so lamely
+ # setup, it is worth printing a warning
+ print "NOTE: We can not locate an INI file for SpamBayes, and the"
+ print "Python for Windows extensions are not installed, meaning we"
+ print "can't locate your 'user' directory. An empty configuration"
+ print "file at '%s' will be used." % optionsPathname.encode('mbcs')
From mhammond at users.sourceforge.net Tue Sep 16 10:01:49 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:01:55 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.11,
1.12
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv1252
Modified Files:
pop3proxy_service.py
Log Message:
Only munge sys.path in source-code versions, and as the service starts,
have it report the username and ini file it is using.
In binary builds, write a log to %temp%\SpamBayesServicen.log
Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** pop3proxy_service.py 9 Sep 2003 01:26:16 -0000 1.11
--- pop3proxy_service.py 16 Sep 2003 14:01:47 -0000 1.12
***************
*** 18,45 ****
import sys, os
! # We are in the 'spambayes\win32' directory. We
! # need the parent on sys.path, so 'spambayes.spambayes' is a package,
! # and 'pop3proxy' is a module
try:
! # module imported by service manager, or 2.3 (in which __main__
! # exists, *and* sys.argv[0] is always already absolute)
! this_filename=__file__
! except NameError:
! this_filename = sys.argv[0]
! if not os.path.isabs(sys.argv[0]):
! # Python 2.3 __main__
! # patch up sys.argv, as our cwd will confuse service registration code
! sys.argv[0] = os.path.abspath(sys.argv[0])
! this_filename = sys.argv[0]
! print sys.argv[0]
! sb_dir = os.path.dirname(os.path.dirname(this_filename))
! sb_scripts_dir = os.path.join(sb_dir,"scripts")
! sys.path.insert(0, sb_dir)
! sys.path.insert(-1, sb_scripts_dir)
! # and change directory here, so pop3proxy uses the default
! # config file etc
! os.chdir(sb_dir)
# Rest of the standard Python modules we use.
--- 18,76 ----
import sys, os
!
! # Messages from pop3proxy will go nowhere when executed as a service
! # Try and detect that print will go nowhere and redirect.
! # redirect output somewhere useful when running as a service.
! import win32api
try:
! win32api.GetConsoleTitle()
! except win32api.error:
! # No console - if we are running from Python sources,
! # redirect to win32traceutil, but if running from a binary
! # install, redirect to a log file.
! # Want to move to logging module later, so for now, we
! # hack together a simple logging strategy.
! if hasattr(sys, "frozen"):
! temp_dir = win32api.GetTempPath()
! for i in range(3,0,-1):
! try: os.unlink(os.path.join(temp_dir, "SpamBayesService%d.log" % (i+1)))
! except os.error: pass
! try:
! os.rename(
! os.path.join(temp_dir, "SpamBayesService%d.log" % i),
! os.path.join(temp_dir, "SpamBayesService%d.log" % (i+1))
! )
! except os.error: pass
! # Open this log, as unbuffered so crashes still get written.
! sys.stdout = open(os.path.join(temp_dir,"SpamBayesService1.log"), "wt", 0)
! sys.stderr = sys.stdout
! else:
! import win32traceutil
! # If running from sources, patch up sys.path
! if not hasattr(sys, "frozen"):
! # We are in the 'spambayes\win32' directory. We
! # need the parent on sys.path, so 'spambayes.spambayes' is a package,
! # and 'pop3proxy' is a module
! try:
! # module imported by service manager, or 2.3 (in which __main__
! # exists, *and* sys.argv[0] is always already absolute)
! this_filename=__file__
! except NameError:
! this_filename = sys.argv[0]
! if not os.path.isabs(sys.argv[0]):
! # Python 2.3 __main__
! # patch up sys.argv, as our cwd will confuse service registration code
! sys.argv[0] = os.path.abspath(sys.argv[0])
! this_filename = sys.argv[0]
! sb_dir = os.path.dirname(os.path.dirname(this_filename))
! sb_scripts_dir = os.path.join(sb_dir,"scripts")
!
! sys.path.insert(0, sb_dir)
! sys.path.insert(-1, sb_scripts_dir)
! # and change directory here, so pop3proxy uses the default
! # config file etc
! os.chdir(sb_dir)
# Rest of the standard Python modules we use.
***************
*** 54,75 ****
import win32serviceutil, win32service
import pywintypes, win32con, winerror
-
from ntsecuritycon import *
- # Messages from pop3proxy will go nowhere when executed as a service
- # Try and detect that print will go nowhere and redirect.
- try:
- # redirect output somewhere useful when running as a service.
- import win32api
- try:
- win32api.GetConsoleTitle()
- except win32api.error:
- # no console - import win32traceutil
- import win32traceutil
- print "popproxy service module loading (as user %s)..." \
- % win32api.GetUserName()
- except ImportError:
- pass
-
class Service(win32serviceutil.ServiceFramework):
# The script name was changed to "sb_server" but I'll leave this as pop3proxy
--- 85,90 ----
***************
*** 88,92 ****
self.event_stopping.set()
sb_server.stop(sb_server.state)
-
def SvcDoRun(self):
--- 103,106 ----
***************
*** 101,109 ****
# Write an event log record - in debug mode we will also
# see this message printed.
import servicemanager
servicemanager.LogMsg(
servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
! (self._svc_name_, '')
)
--- 115,127 ----
# Write an event log record - in debug mode we will also
# see this message printed.
+ from spambayes.Options import optionsPathname
+ extra = " as user '%s', using config file '%s'" \
+ % (win32api.GetUserNameEx(2).encode("mbcs"),
+ optionsPathname.encode("mbcs"))
import servicemanager
servicemanager.LogMsg(
servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
! (self._svc_name_, extra)
)
From mhammond at users.sourceforge.net Tue Sep 16 10:13:12 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:13:20 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.74,1.75
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv3835
Modified Files:
Options.py
Log Message:
Might as well beat our users to testing with usernames/paths that include
extended characters.
While I was at it, might as well ensure the "per user" directory exists!
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.74
retrieving revision 1.75
diff -C2 -d -r1.74 -r1.75
*** Options.py 16 Sep 2003 14:00:12 -0000 1.74
--- Options.py 16 Sep 2003 14:13:08 -0000 1.75
***************
*** 984,988 ****
shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
"SpamBayes", "Proxy")
! optionsPathname = os.path.join(windowsUserDirectory, 'bayescustomize.ini')
except ImportError:
# We are on Windows, with no BAYESCUSTOMIZE set, no ini file
--- 984,997 ----
shell.SHGetFolderPath(0,shellcon.CSIDL_APPDATA,0,0),
"SpamBayes", "Proxy")
! try:
! if not os.path.isdir(windowsUserDirectory):
! os.makedirs(windowsUserDirectory)
! optionsPathname = os.path.join(windowsUserDirectory,
! 'bayescustomize.ini')
! # Not everyone is unicode aware - keep it a string.
! optionsPathname = optionsPathname.encode("mbcs")
! except os.error:
! # unable to make the directory - stick to default.
! pass
except ImportError:
# We are on Windows, with no BAYESCUSTOMIZE set, no ini file
From mhammond at users.sourceforge.net Tue Sep 16 10:14:13 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 10:14:16 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.12,
1.13
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv4080
Modified Files:
pop3proxy_service.py
Log Message:
Pathname will always be a string (rather than Unicode) for the forseeable
future in the proxy.
Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** pop3proxy_service.py 16 Sep 2003 14:01:47 -0000 1.12
--- pop3proxy_service.py 16 Sep 2003 14:14:11 -0000 1.13
***************
*** 118,122 ****
extra = " as user '%s', using config file '%s'" \
% (win32api.GetUserNameEx(2).encode("mbcs"),
! optionsPathname.encode("mbcs"))
import servicemanager
servicemanager.LogMsg(
--- 118,122 ----
extra = " as user '%s', using config file '%s'" \
% (win32api.GetUserNameEx(2).encode("mbcs"),
! optionsPathname)
import servicemanager
servicemanager.LogMsg(
From anadelonbrin at users.sourceforge.net Tue Sep 16 19:51:14 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 19:51:24 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs dialog_map.py,
1.33, 1.34
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs
In directory sc8-pr-cvs1:/tmp/cvs-serv30309/Outlook2000/dialogs
Modified Files:
dialog_map.py
Log Message:
Add a warning for those with highly (>5 times) imbalanced ham and spam.
** I don't have something to tinker with the dialog layout - could someone that does please
make room for the error message? **
Index: dialog_map.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/dialog_map.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** dialog_map.py 5 Sep 2003 11:50:57 -0000 1.33
--- dialog_map.py 16 Sep 2003 23:51:12 -0000 1.34
***************
*** 37,40 ****
--- 37,53 ----
if nspam > 10 and nham > 10:
db_status = "Database has %d good and %d spam." % (nham, nspam)
+ db_ratio = nham/float(nspam)
+ big = small = None
+ if db_ratio > 5.0:
+ big = "ham"
+ small = "spam"
+ elif db_ratio < (1/5.0):
+ big = "spam"
+ small = "ham"
+ if big is not None:
+ db_status = "%s\nWarning: you have much more %s than %s - " \
+ "SpamBayes works best with approximately even " \
+ "numbers of ham and spam." % (db_status, big,
+ small)
elif nspam > 0 or nham > 0:
db_status = "Database only has %d good and %d spam - you should " \
From anadelonbrin at users.sourceforge.net Tue Sep 16 20:03:56 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Tue Sep 16 20:03:59 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py,1.4,1.5
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv32667/scripts
Modified Files:
sb_server.py
Log Message:
Fix [ spambayes-Bugs-806632 ] sb_server failure when saving config
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** sb_server.py 13 Sep 2003 05:10:49 -0000 1.4
--- sb_server.py 17 Sep 2003 00:03:54 -0000 1.5
***************
*** 722,729 ****
del proxyListeners[:]
! # Close the database; we should anyway, and gdbm complains if we
! # try to reopen it without closing it first.
! state.bayes.store()
! state.bayes.close()
state = State()
--- 722,730 ----
del proxyListeners[:]
! # Close the database (if there is one); we should anyway, and gdbm
! # complains if we try to reopen it without closing it first.
! if hasattr(state, "bayes"):
! state.bayes.store()
! state.bayes.close()
state = State()
From mhammond at users.sourceforge.net Tue Sep 16 20:11:46 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Tue Sep 16 20:11:49 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/dialogs/resources
dialogs.rc, 1.38, 1.39
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv1872
Modified Files:
dialogs.rc
Log Message:
Give the database status room for a 3rd line.
Index: dialogs.rc
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/dialogs/resources/dialogs.rc,v
retrieving revision 1.38
retrieving revision 1.39
diff -C2 -d -r1.38 -r1.39
*** dialogs.rc 8 Sep 2003 07:53:05 -0000 1.38
--- dialogs.rc 17 Sep 2003 00:11:44 -0000 1.39
***************
*** 490,504 ****
LTEXT "SpamBayes Version Here",IDC_VERSION,6,54,242,8
LTEXT "SpamBayes requires training before it is effective. Click on the 'Training' tab, or use the Configuration Wizard to train.",
! IDC_STATIC,6,70,242,17
! LTEXT "Training database status:",IDC_STATIC,6,97,222,8
! LTEXT "123 spam messages; 456 good messages",
! IDC_TRAINING_STATUS,6,108,242,19,SS_SUNKEN
CONTROL "Enable SpamBayes",IDC_BUT_FILTER_ENABLE,"Button",
BS_AUTOCHECKBOX | WS_TABSTOP,6,173,97,11
LTEXT "Certain spam is moved to Folder1\nPossible spam is moved too",
! IDC_FILTER_STATUS,6,147,242,19,SS_SUNKEN
PUSHBUTTON "Reset Configuration...",IDC_BUT_RESET,6,190,84,15
PUSHBUTTON "Configuration Wizard...",IDC_BUT_WIZARD,164,190,84,15
! LTEXT "Filter status:",IDC_STATIC,6,136,222,8
CONTROL 1062,IDC_LOGO_GRAPHIC,"Static",SS_BITMAP |
SS_REALSIZEIMAGE,0,2,275,52
--- 490,504 ----
LTEXT "SpamBayes Version Here",IDC_VERSION,6,54,242,8
LTEXT "SpamBayes requires training before it is effective. Click on the 'Training' tab, or use the Configuration Wizard to train.",
! IDC_STATIC,6,67,242,17
! LTEXT "Training database status:",IDC_STATIC,6,90,222,8
! LTEXT "123 spam messages; 456 good messages\r\nLine2\r\nLine3",
! IDC_TRAINING_STATUS,6,101,242,27,SS_SUNKEN
CONTROL "Enable SpamBayes",IDC_BUT_FILTER_ENABLE,"Button",
BS_AUTOCHECKBOX | WS_TABSTOP,6,173,97,11
LTEXT "Certain spam is moved to Folder1\nPossible spam is moved too",
! IDC_FILTER_STATUS,6,146,242,19,SS_SUNKEN
PUSHBUTTON "Reset Configuration...",IDC_BUT_RESET,6,190,84,15
PUSHBUTTON "Configuration Wizard...",IDC_BUT_WIZARD,164,190,84,15
! LTEXT "Filter status:",IDC_STATIC,6,135,222,8
CONTROL 1062,IDC_LOGO_GRAPHIC,"Static",SS_BITMAP |
SS_REALSIZEIMAGE,0,2,275,52
From anadelonbrin at users.sourceforge.net Wed Sep 17 23:59:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 17 23:59:07 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_pop3dnd.py, 1.2,
1.3 sb_server.py, 1.5, 1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv22711/scripts
Modified Files:
sb_pop3dnd.py sb_server.py
Log Message:
If we can't find a config file anywhere but the windows app data directory, then load it.
Swap various over to correct places in prep for 1.0a6. If you were using notate_to, notate_subject, hammie:debug_header, or many of the pop3proxy storage options, you will need to update your config file. These are in the most suitable places now.
Use urllib not urllib2 to shut down the proxy. Why? Because it works and the other doesn't.
Create the server strings for the ui *after* reading in the command line parameters.
Fix importing Options.py in ProxyUI.py for the binary.
Index: sb_pop3dnd.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_pop3dnd.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_pop3dnd.py 6 Sep 2003 04:03:34 -0000 1.2
--- sb_pop3dnd.py 18 Sep 2003 03:58:59 -0000 1.3
***************
*** 863,872 ****
app = Application("SpambayesIMAPServer")
! spam_box = SpambayesMailbox("Spam", 0, options["imapserver",
! "spam_directory"])
! unsure_box = SpambayesMailbox("Unsure", 1, options["imapserver",
! "unsure_directory"])
ham_train_box = SpambayesMailbox("TrainAsHam", 2,
! options["imapserver", "ham_directory"])
spam_trainer = Trainer(spam_box, True)
--- 863,872 ----
app = Application("SpambayesIMAPServer")
! spam_box = SpambayesMailbox("Spam", 0, options["Storage",
! "spam_cache"])
! unsure_box = SpambayesMailbox("Unsure", 1, options["Storage",
! "unknown_cache"])
ham_train_box = SpambayesMailbox("TrainAsHam", 2,
! options["Storage", "ham_cache"])
spam_trainer = Trainer(spam_box, True)
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** sb_server.py 17 Sep 2003 00:03:54 -0000 1.5
--- sb_server.py 18 Sep 2003 03:58:59 -0000 1.6
***************
*** 461,469 ****
isSuppressedBulkHam = \
(cls == options["Headers", "header_ham_string"] and
! options["pop3proxy", "no_cache_bulk_ham"] and
msg.get('precedence') in ['bulk', 'list'])
# Suppress large messages if the options say so.
! size_limit = options["pop3proxy",
"no_cache_large_messages"]
isTooBig = size_limit > 0 and \
--- 461,469 ----
isSuppressedBulkHam = \
(cls == options["Headers", "header_ham_string"] and
! options["Storage", "no_cache_bulk_ham"] and
msg.get('precedence') in ['bulk', 'list'])
# Suppress large messages if the options say so.
! size_limit = options["Storage",
"no_cache_large_messages"]
isTooBig = size_limit > 0 and \
***************
*** 473,477 ****
# messages or suppressed bulk ham.
if (not state.isTest and
! options["pop3proxy", "cache_messages"] and
not isSuppressedBulkHam and not isTooBig):
# Write the message into the Unknown cache.
--- 473,477 ----
# messages or suppressed bulk ham.
if (not state.isTest and
! options["Storage", "cache_messages"] and
not isSuppressedBulkHam and not isTooBig):
# Write the message into the Unknown cache.
***************
*** 586,591 ****
self.uiPort = options["html_ui", "port"]
self.launchUI = options["html_ui", "launch_browser"]
! self.gzipCache = options["pop3proxy", "cache_use_gzip"]
! self.cacheExpiryDays = options["pop3proxy", "cache_expiry_days"]
self.runTestServer = False
self.isTest = False
--- 586,591 ----
self.uiPort = options["html_ui", "port"]
self.launchUI = options["html_ui", "launch_browser"]
! self.gzipCache = options["Storage", "cache_use_gzip"]
! self.cacheExpiryDays = options["Storage", "cache_expiry_days"]
self.runTestServer = False
self.isTest = False
***************
*** 634,657 ****
# Create/open the Corpuses. Use small cache sizes to avoid hogging
# lots of memory.
! map(ensureDir, [options["pop3proxy", "spam_cache"],
! options["pop3proxy", "ham_cache"],
! options["pop3proxy", "unknown_cache"]])
if self.gzipCache:
factory = GzipFileMessageFactory()
else:
factory = FileMessageFactory()
! age = options["pop3proxy", "cache_expiry_days"]*24*60*60
self.spamCorpus = ExpiryFileCorpus(age, factory,
! options["pop3proxy",
"spam_cache"],
'[0123456789\-]*',
cacheSize=20)
self.hamCorpus = ExpiryFileCorpus(age, factory,
! options["pop3proxy",
"ham_cache"],
'[0123456789\-]*',
cacheSize=20)
self.unknownCorpus = ExpiryFileCorpus(age, factory,
! options["pop3proxy",
"unknown_cache"],
'[0123456789\-]*',
--- 634,657 ----
# Create/open the Corpuses. Use small cache sizes to avoid hogging
# lots of memory.
! map(ensureDir, [options["Storage", "spam_cache"],
! options["Storage", "ham_cache"],
! options["Storage", "unknown_cache"]])
if self.gzipCache:
factory = GzipFileMessageFactory()
else:
factory = FileMessageFactory()
! age = options["Storage", "cache_expiry_days"]*24*60*60
self.spamCorpus = ExpiryFileCorpus(age, factory,
! options["Storage",
"spam_cache"],
'[0123456789\-]*',
cacheSize=20)
self.hamCorpus = ExpiryFileCorpus(age, factory,
! options["Storage",
"ham_cache"],
'[0123456789\-]*',
cacheSize=20)
self.unknownCorpus = ExpiryFileCorpus(age, factory,
! options["Storage",
"unknown_cache"],
'[0123456789\-]*',
***************
*** 766,771 ****
# Shutdown as though through the web UI. This will save the DB, allow
# any open proxy connections to complete, etc.
! from urllib2 import urlopen
! from urllib import urlencode
urlopen('http://localhost:%d/save' % state.uiPort,
urlencode({'how': 'Save & shutdown'})).read()
--- 766,770 ----
# Shutdown as though through the web UI. This will save the DB, allow
# any open proxy connections to complete, etc.
! from urllib import urlopen, urlencode
urlopen('http://localhost:%d/save' % state.uiPort,
urlencode({'how': 'Save & shutdown'})).read()
***************
*** 810,815 ****
print "and engine %s.\n" % (get_version_string(),)
- prepare(state=state)
-
if 0 <= len(args) <= 2:
# Normal usage, with optional server name and port number.
--- 809,812 ----
***************
*** 823,826 ****
--- 820,824 ----
state.proxyPorts = [('', 110)]
+ prepare(state=state)
start(state=state)
From anadelonbrin at users.sourceforge.net Wed Sep 17 23:59:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Wed Sep 17 23:59:10 2003
Subject: [Spambayes-checkins] spambayes/spambayes ImapUI.py, 1.17,
1.18 Options.py, 1.75, 1.76 ProxyUI.py, 1.21, 1.22 Version.py,
1.22, 1.23 hammie.py, 1.10, 1.11 message.py, 1.36, 1.37
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv22711/spambayes
Modified Files:
ImapUI.py Options.py ProxyUI.py Version.py hammie.py
message.py
Log Message:
If we can't find a config file anywhere but the windows app data directory, then load it.
Swap various over to correct places in prep for 1.0a6. If you were using notate_to, notate_subject, hammie:debug_header, or many of the pop3proxy storage options, you will need to update your config file. These are in the most suitable places now.
Use urllib not urllib2 to shut down the proxy. Why? Because it works and the other doesn't.
Create the server strings for the ui *after* reading in the command line parameters.
Fix importing Options.py in ProxyUI.py for the binary.
Index: ImapUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ImapUI.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** ImapUI.py 26 Aug 2003 04:30:41 -0000 1.17
--- ImapUI.py 18 Sep 2003 03:58:59 -0000 1.18
***************
*** 68,73 ****
('html_ui', 'allow_remote_connections'),
('Header Options', None),
! ('pop3proxy', 'notate_to'),
! ('pop3proxy', 'notate_subject'),
('Storage Options', None),
('Storage', 'persistent_storage_file'),
--- 68,73 ----
('html_ui', 'allow_remote_connections'),
('Header Options', None),
! ('Headers', 'notate_to'),
! ('Headers', 'notate_subject'),
('Storage Options', None),
('Storage', 'persistent_storage_file'),
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.75
retrieving revision 1.76
diff -C2 -d -r1.75 -r1.76
*** Options.py 16 Sep 2003 14:13:08 -0000 1.75
--- Options.py 18 Sep 2003 03:58:59 -0000 1.76
***************
*** 419,432 ****
"Hammie": (
- ("debug_header", "Add debug header", False,
- """Enable debugging information in the header.""",
- BOOLEAN, RESTORE),
-
- ("debug_header_name", "Debug header name", "X-Spambayes-Debug",
- """Name of a debugging header for spambayes hackers, showing the
- strongest clues that have resulted in the classification in the
- standard header.""",
- HEADER_NAME, RESTORE),
-
("train_on_filter", "Train when filtering", False,
"""Train when filtering? After filtering a message, hammie can then
--- 419,422 ----
***************
*** 470,473 ****
--- 460,517 ----
working directory.""",
FILE_WITH_PATH, DO_NOT_RESTORE),
+
+ ("cache_use_gzip", "Use gzip", False,
+ """Use gzip to compress the cache.""",
+ BOOLEAN, RESTORE),
+
+ ("cache_expiry_days", "Days before cached messages expire", 7,
+ """Messages will be expired from the cache after this many days.
+ After this time, you will no longer be able to train on these messages
+ (note this does not effect the copy of the message that you have in
+ your mail client).""",
+ INTEGER, RESTORE),
+
+ ("spam_cache", "Spam cache directory", "pop3proxy-spam-cache",
+ """Directory that SpamBayes should cache spam in. If this does
+ not exist, it will be created.""",
+ PATH, DO_NOT_RESTORE),
+
+ ("ham_cache", "Ham cache directory", "pop3proxy-ham-cache",
+ """Directory that SpamBayes should cache ham in. If this does
+ not exist, it will be created.""",
+ PATH, DO_NOT_RESTORE),
+
+ ("unknown_cache", "Unknown cache directory", "pop3proxy-unknown-cache",
+ """Directory that SpamBayes should cache unclassified messages in.
+ If this does not exist, it will be created.""",
+ PATH, DO_NOT_RESTORE),
+
+ ("cache_messages", "Cache messages", True,
+ """You can disable the pop3proxy caching of messages. This
+ will make the proxy a bit faster, and make it use less space
+ on your hard drive. The proxy uses its cache for reviewing
+ and training of messages, so if you disable caching you won't
+ be able to do further training unless you re-enable it.
+ Thus, you should only turn caching off when you are satisfied
+ with the filtering that Spambayes is doing for you.""",
+ BOOLEAN, RESTORE),
+
+ ("no_cache_bulk_ham", "Suppress caching of bulk ham", False,
+ """Where message caching is enabled, this option suppresses caching
+ of messages which are classified as ham and marked as
+ 'Precedence: bulk' or 'Precedence: list'. If you subscribe to a
+ high-volume mailing list then your 'Review messages' page can be
+ overwhelmed with list messages, making training a pain. Once you've
+ trained Spambayes on enough list traffic, you can use this option
+ to prevent that traffic showing up in 'Review messages'.""",
+ BOOLEAN, RESTORE),
+
+ ("no_cache_large_messages", "Maximum size of cached messages", 0,
+ """Where message caching is enabled, this option suppresses caching
+ of messages which are larger than this value (measured in bytes).
+ If you receive a lot of messages that include large attachments
+ (and are correctly classified), you may not wish to cache these.
+ If you set this to zero (0), then this option will have no effect.""",
+ INTEGER, RESTORE),
),
***************
*** 588,591 ****
--- 632,656 ----
option adds this information to a header added to each message.""",
BOOLEAN, RESTORE),
+
+ ("notate_to", "Notate to", (),
+ """Some email clients (Outlook Express, for example) can only set up
+ filtering rules on a limited set of headers. These clients cannot
+ test for the existence/value of an arbitrary header and filter mail
+ based on that information. To accommodate these kind of mail clients,
+ you can add "spam", "ham", or "unsure" to the recipient list. A
+ filter rule can then use this to see if one of these words (followed
+ by a comma) is in the recipient list, and route the mail to an
+ appropriate folder, or take whatever other action is supported and
+ appropriate for the mail classification.
+
+ As it interferes with replying, you may only wish to do this for
+ spam messages; simply tick the boxes of the classifications take
+ should be identified in this fashion.""",
+ ("ham", "spam", "unsure"), RESTORE),
+
+ ("notate_subject", "Classify in subject: header", (),
+ """This option will add the same information as 'Notate To',
+ but to the start of the mail subject line.""",
+ ("ham", "spam", "unsure"), RESTORE),
),
***************
*** 617,699 ****
specify the same number of ports as servers, separated by commas.""",
SERVER, DO_NOT_RESTORE),
-
- ("cache_use_gzip", "Use gzip", False,
- """Use gzip to compress the cache.""",
- BOOLEAN, RESTORE),
-
- ("cache_expiry_days", "Days before cached messages expire", 7,
- """Messages will be expired from the cache after this many days.
- After this time, you will no longer be able to train on these messages
- (note this does not effect the copy of the message that you have in
- your mail client).""",
- INTEGER, RESTORE),
-
- ("spam_cache", "Spam cache directory", "pop3proxy-spam-cache",
- """Directory that SpamBayes should cache spam in. If this does
- not exist, it will be created.""",
- PATH, DO_NOT_RESTORE),
-
- ("ham_cache", "Ham cache directory", "pop3proxy-ham-cache",
- """Directory that SpamBayes should cache ham in. If this does
- not exist, it will be created.""",
- PATH, DO_NOT_RESTORE),
-
- ("unknown_cache", "Unknown cache directory", "pop3proxy-unknown-cache",
- """Directory that SpamBayes should cache unclassified messages in.
- If this does not exist, it will be created.""",
- PATH, DO_NOT_RESTORE),
-
- ("notate_to", "Notate to", (),
- """Some email clients (Outlook Express, for example) can only set up
- filtering rules on a limited set of headers. These clients cannot
- test for the existence/value of an arbitrary header and filter mail
- based on that information. To accommodate these kind of mail clients,
- you can add "spam", "ham", or "unsure" to the recipient list. A
- filter rule can then use this to see if one of these words (followed
- by a comma) is in the recipient list, and route the mail to an
- appropriate folder, or take whatever other action is supported and
- appropriate for the mail classification.
-
- As it interferes with replying, you may only wish to do this for
- spam messages; simply tick the boxes of the classifications take
- should be identified in this fashion.""",
- ("ham", "spam", "unsure"), RESTORE),
-
- ("notate_subject", "Classify in subject: header", (),
- """This option will add the same information as 'Notate To',
- but to the start of the mail subject line.""",
- ("ham", "spam", "unsure"), RESTORE),
-
- ("cache_messages", "Cache messages", True,
- """You can disable the pop3proxy caching of messages. This
- will make the proxy a bit faster, and make it use less space
- on your hard drive. The proxy uses its cache for reviewing
- and training of messages, so if you disable caching you won't
- be able to do further training unless you re-enable it.
- Thus, you should only turn caching off when you are satisfied
- with the filtering that Spambayes is doing for you.""",
- BOOLEAN, RESTORE),
-
- ("no_cache_bulk_ham", "Suppress caching of bulk ham", False,
- """Where message caching is enabled, this option suppresses caching
- of messages which are classified as ham and marked as
- 'Precedence: bulk' or 'Precedence: list'. If you subscribe to a
- high-volume mailing list then your 'Review messages' page can be
- overwhelmed with list messages, making training a pain. Once you've
- trained Spambayes on enough list traffic, you can use this option
- to prevent that traffic showing up in 'Review messages'.""",
- BOOLEAN, RESTORE),
-
- ("no_cache_large_messages", "Maximum size of cached messages", 0,
- """Where message caching is enabled, this option suppresses caching
- of messages which are larger than this value (measured in bytes).
- If you receive a lot of messages that include large attachments
- (and are correctly classified), you may not wish to cache these.
- If you set this to zero (0), then this option will have no effect.""",
- INTEGER, RESTORE),
),
"smtpproxy" : (
-
("remote_servers", "Remote Servers", (),
"""The Spambayes SMTP proxy intercepts outgoing email - if you
--- 682,688 ----
***************
*** 903,918 ****
"""The port to serve the SpamBayes IMAP server on.""",
PORT, RESTORE),
-
- ("spam_directory", "Spam directory", "imapserver-spam",
- """The directory to store spam messages in.""",
- PATH, DO_NOT_RESTORE),
-
- ("ham_directory", "Ham directory", "imapserver-ham",
- """The directory to store ham messages in.""",
- PATH, DO_NOT_RESTORE),
-
- ("unsure_directory", "Unsure directory", "imapserver-unsure",
- """The directory to store unsure messages in.""",
- PATH, DO_NOT_RESTORE),
),
--- 892,895 ----
***************
*** 991,994 ****
--- 968,983 ----
# Not everyone is unicode aware - keep it a string.
optionsPathname = optionsPathname.encode("mbcs")
+ # If the file exists, then load it.
+ if os.path.exists(optionsPathname):
+ options.merge_file(optionsPathname)
+ else:
+ # If the file doesn't exist, then let's get the user to
+ # store their databases here as well, by default, and
+ # save the file.
+ db_name = os.path.join(windowsUserDirectory,
+ "statistics_database.db").\
+ encode("mbcs")
+ options["Storage", "persistent_storage_file"] = db_name
+ options.update_file(optionsPathname)
except os.error:
# unable to make the directory - stick to default.
Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** ProxyUI.py 9 Sep 2003 06:05:15 -0000 1.21
--- ProxyUI.py 18 Sep 2003 03:58:59 -0000 1.22
***************
*** 68,72 ****
import tokenizer
import UserInterface
! from Options import options
import spambayes.mboxutils
from email.Iterators import typed_subpart_iterator
--- 68,72 ----
import tokenizer
import UserInterface
! from spambayes.Options import options
import spambayes.mboxutils
from email.Iterators import typed_subpart_iterator
***************
*** 89,94 ****
('html_ui', 'http_password'),
('Header Options', None),
! ('pop3proxy', 'notate_to'),
! ('pop3proxy', 'notate_subject'),
('SMTP Proxy Options', None),
('smtpproxy', 'remote_servers'),
--- 89,94 ----
('html_ui', 'http_password'),
('Header Options', None),
! ('Headers', 'notate_to'),
! ('Headers', 'notate_subject'),
('SMTP Proxy Options', None),
('smtpproxy', 'remote_servers'),
***************
*** 100,106 ****
('Storage', 'persistent_storage_file'),
('Storage', 'messageinfo_storage_file'),
! ('pop3proxy', 'cache_messages'),
! ('pop3proxy', 'no_cache_bulk_ham'),
! ('pop3proxy', 'no_cache_large_messages'),
('Statistics Options', None),
('Categorization', 'ham_cutoff'),
--- 100,106 ----
('Storage', 'persistent_storage_file'),
('Storage', 'messageinfo_storage_file'),
! ('Storage', 'cache_messages'),
! ('Storage', 'no_cache_bulk_ham'),
! ('Storage', 'no_cache_large_messages'),
('Statistics Options', None),
('Categorization', 'ham_cutoff'),
***************
*** 126,134 ****
('Storage Options', None),
('Storage', 'persistent_use_database'),
! ('pop3proxy', 'cache_expiry_days'),
! ('pop3proxy', 'cache_use_gzip'),
! ('pop3proxy', 'ham_cache'),
! ('pop3proxy', 'spam_cache'),
! ('pop3proxy', 'unknown_cache'),
('Tokenising Options', None),
('Tokenizer', 'extract_dow'),
--- 126,134 ----
('Storage Options', None),
('Storage', 'persistent_use_database'),
! ('Storage', 'cache_expiry_days'),
! ('Storage', 'cache_use_gzip'),
! ('Storage', 'ham_cache'),
! ('Storage', 'spam_cache'),
! ('Storage', 'unknown_cache'),
('Tokenising Options', None),
('Tokenizer', 'extract_dow'),
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** Version.py 15 Sep 2003 01:00:18 -0000 1.22
--- Version.py 18 Sep 2003 03:58:59 -0000 1.23
***************
*** 90,94 ****
"Date": "September 2003",
"InterfaceVersion": 0.02,
! "InterfaceDescription": "SpamBayes IMAP Filter Web Interface Alpha2",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
--- 90,94 ----
"Date": "September 2003",
"InterfaceVersion": 0.02,
! "InterfaceDescription": "SpamBayes IMAP Server Web Interface Alpha2",
"Full Description": """%(Description)s, version %(Version)s (%(Date)s),
using %(InterfaceDescription)s, version %(InterfaceVersion)s""",
Index: hammie.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammie.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** hammie.py 10 Sep 2003 04:12:20 -0000 1.10
--- hammie.py 18 Sep 2003 03:58:59 -0000 1.11
***************
*** 85,89 ****
Returns the same message with a new disposition header.
-
"""
--- 85,88 ----
***************
*** 95,101 ****
ham_cutoff = options["Categorization", "ham_cutoff"]
if debugheader == None:
! debugheader = options["Hammie", "debug_header_name"]
if debug == None:
! debug = options["Hammie", "debug_header"]
if train == None:
train = options["Hammie", "train_on_filter"]
--- 94,100 ----
ham_cutoff = options["Categorization", "ham_cutoff"]
if debugheader == None:
! debugheader = options["Headers", "evidence_header_name"]
if debug == None:
! debug = options["Headers", "include_evidence"]
if train == None:
train = options["Hammie", "train_on_filter"]
Index: message.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/message.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -C2 -d -r1.36 -r1.37
*** message.py 8 Sep 2003 14:46:57 -0000 1.36
--- message.py 18 Sep 2003 03:58:59 -0000 1.37
***************
*** 321,325 ****
# allow filtering in 'stripped down' mailers like Outlook Express,
# so for the moment, they stay in.
! if disposition in options["pop3proxy", "notate_to"]:
try:
self.replace_header("To", "%s,%s" % (disposition,
--- 321,325 ----
# allow filtering in 'stripped down' mailers like Outlook Express,
# so for the moment, they stay in.
! if disposition in options["Headers", "notate_to"]:
try:
self.replace_header("To", "%s,%s" % (disposition,
***************
*** 328,332 ****
self["To"] = disposition
! if disposition in options["pop3proxy", "notate_subject"]:
try:
self.replace_header("Subject", "%s,%s" % (disposition,
--- 328,332 ----
self["To"] = disposition
! if disposition in options["Headers", "notate_subject"]:
try:
self.replace_header("Subject", "%s,%s" % (disposition,
From anadelonbrin at users.sourceforge.net Thu Sep 18 00:12:53 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:12:58 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.76,1.77
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv24992/spambayes
Modified Files:
Options.py
Log Message:
Store the user's caches and messageinfo.db file in the app data folder as well (by default),
if we have everything else there.
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.76
retrieving revision 1.77
diff -C2 -d -r1.76 -r1.77
*** Options.py 18 Sep 2003 03:58:59 -0000 1.76
--- Options.py 18 Sep 2003 04:12:51 -0000 1.77
***************
*** 973,982 ****
else:
# If the file doesn't exist, then let's get the user to
! # store their databases here as well, by default, and
! # save the file.
db_name = os.path.join(windowsUserDirectory,
! "statistics_database.db").\
! encode("mbcs")
! options["Storage", "persistent_storage_file"] = db_name
options.update_file(optionsPathname)
except os.error:
--- 973,995 ----
else:
# If the file doesn't exist, then let's get the user to
! # store their databases and caches here as well, by
! # default, and save the file.
db_name = os.path.join(windowsUserDirectory,
! "statistics_database.db")
! mi_db = os.path.join(windowsUserDirectory,
! "message_info_database.db")
! h_cache = os.path.join(windowsUserDirectory,
! "ham_cache").encode("mbcs")
! u_cache = os.path.join(windowsUserDirectory,
! "unknown_cache").encode("mbcs")
! s_cache = os.path.join(windowsUserDirectory,
! "spam_cache").encode("mbcs")
! options["Storage", "spam_cache"] = s_cache
! options["Storage", "ham_cache"] = h_cache
! options["Storage", "unknown_cache"] = u_cache
! options["Storage", "persistent_storage_file"] = \
! db_name.encode("mbcs")
! options["Storage", "messageinfo_storage_file"] = \
! mi_db.encode("mbcs")
options.update_file(optionsPathname)
except os.error:
From mhammond at users.sourceforge.net Thu Sep 18 00:15:27 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 00:15:33 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_service.py, 1.13,
1.14
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv25317
Modified Files:
pop3proxy_service.py
Log Message:
There is no good reason not to use win32api.GetUserName()
Index: pop3proxy_service.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_service.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** pop3proxy_service.py 16 Sep 2003 14:14:11 -0000 1.13
--- pop3proxy_service.py 18 Sep 2003 04:15:24 -0000 1.14
***************
*** 117,121 ****
from spambayes.Options import optionsPathname
extra = " as user '%s', using config file '%s'" \
! % (win32api.GetUserNameEx(2).encode("mbcs"),
optionsPathname)
import servicemanager
--- 117,121 ----
from spambayes.Options import optionsPathname
extra = " as user '%s', using config file '%s'" \
! % (win32api.GetUserName(),
optionsPathname)
import servicemanager
From anadelonbrin at users.sourceforge.net Thu Sep 18 00:31:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:31:04 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000
default_bayes_customize.ini, 1.8, 1.9
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv27180/Outlook2000
Modified Files:
default_bayes_customize.ini
Log Message:
Change the Outlook plug-in to use the general default (currently False) for the
experimental_ham_spam_imbalance adjustment.
Change the default for address_headers to include to, cc, reply-to, and sender as per
Tim's suggestion.
Index: default_bayes_customize.ini
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/default_bayes_customize.ini,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** default_bayes_customize.ini 15 Sep 2003 01:08:11 -0000 1.8
--- default_bayes_customize.ini 18 Sep 2003 04:30:59 -0000 1.9
***************
*** 16,26 ****
record_header_absence: True
- # These should help. All but "from" are disabled by default, because
- # they're killer-good clues for bad reasons when using mixed-source
- # data.
- address_headers: from to cc sender reply-to
-
-
[Classifier]
# This will probably go away if testing confirms it's a Good Thing.
! experimental_ham_spam_imbalance_adjustment: True
--- 16,22 ----
record_header_absence: True
[Classifier]
# This will probably go away if testing confirms it's a Good Thing.
! # Testing seems to be indicating that this is a Bad Thing, so we
! # leave it disabled.
! #experimental_ham_spam_imbalance_adjustment: True
From anadelonbrin at users.sourceforge.net Thu Sep 18 00:31:01 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 00:31:06 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py,1.77,1.78
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27180/spambayes
Modified Files:
Options.py
Log Message:
Change the Outlook plug-in to use the general default (currently False) for the
experimental_ham_spam_imbalance adjustment.
Change the default for address_headers to include to, cc, reply-to, and sender as per
Tim's suggestion.
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.77
retrieving revision 1.78
diff -C2 -d -r1.77 -r1.78
*** Options.py 18 Sep 2003 04:12:51 -0000 1.77
--- Options.py 18 Sep 2003 04:30:59 -0000 1.78
***************
*** 101,105 ****
BOOLEAN, RESTORE),
! ("address_headers", "Address headers to mine", ("from",),
"""Mine the following address headers. If you have mixed source
corpuses (as opposed to a mixed sauce walrus, which is delicious!)
--- 101,105 ----
BOOLEAN, RESTORE),
! ("address_headers", "Address headers to mine", ("from", "to", "cc", "sender", "reply-to"),
"""Mine the following address headers. If you have mixed source
corpuses (as opposed to a mixed sauce walrus, which is delicious!)
From mhammond at users.sourceforge.net Thu Sep 18 01:36:41 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 01:36:45 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 about.html,1.23,1.24
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv3161
Modified Files:
about.html
Log Message:
Remove an accidental
tag.
Index: about.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/about.html,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** about.html 10 Sep 2003 07:33:39 -0000 1.23
--- about.html 18 Sep 2003 05:36:39 -0000 1.24
***************
*** 41,45 ****
When Outlook is under load, SpamBayes may occasionally miss some
messages (as Outlook doesn't tell us they appeared). Also, if you
! use builtin Outlook rules, you may find that occasionally they also
fail to filter messages (as both SpamBayes and the builtin rules
"fight" over processing the message). Both of these
--- 41,45 ----
When Outlook is under load, SpamBayes may occasionally miss some
messages (as Outlook doesn't tell us they appeared). Also, if you
! use builtin Outlook rules, you may find that occasionally they also
fail to filter messages (as both SpamBayes and the builtin rules
"fight" over processing the message). Both of these
From montanaro at users.sourceforge.net Thu Sep 18 09:55:14 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 09:55:22 2003
Subject: [Spambayes-checkins] spambayes/spambayes Options.py, 1.78,
1.79 ProxyUI.py, 1.22, 1.23
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27691/spambayes
Modified Files:
Options.py ProxyUI.py
Log Message:
Place a threshold on the number of items displayed per section. The default
is 10,000, large enough so most people shouldn't notice the change. If you
have a large amount of untrained mail, setting a smallish (100 or so) value
for the "rows_per_section" variable in the "html_ui" section of your options
file gives you a reasonable amount of mail to consider. (I personally go
numb after scanning a couple dozen subjects.) It also means that if you
click one of the section headers to set all the radio buttons to a new value
a slow JavaScript implementation won't have you waiting for the next
Spammish Inquisition to arrive.
Index: Options.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Options.py,v
retrieving revision 1.78
retrieving revision 1.79
diff -C2 -d -r1.78 -r1.79
*** Options.py 18 Sep 2003 04:30:59 -0000 1.78
--- Options.py 18 Sep 2003 13:55:11 -0000 1.79
***************
*** 785,788 ****
--- 785,793 ----
authorized user password here.""",
r"[\w]+", RESTORE),
+
+ ("rows_per_section", "Rows per section", 10000,
+ """Number of rows to display per ham/spam/unsure section.""",
+ INTEGER, RESTORE),
+
),
Index: ProxyUI.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/ProxyUI.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** ProxyUI.py 18 Sep 2003 03:58:59 -0000 1.22
--- ProxyUI.py 18 Sep 2003 13:55:11 -0000 1.23
***************
*** 249,253 ****
if not options["html_ui", "display_to"]:
del table.to_header
! for key, messageInfo in keyedMessageInfo:
row = self.html.reviewRow.clone()
if label == 'Spam':
--- 249,254 ----
if not options["html_ui", "display_to"]:
del table.to_header
! nrows = options["html_ui", "rows_per_section"]
! for key, messageInfo in keyedMessageInfo[:nrows]:
row = self.html.reviewRow.clone()
if label == 'Spam':
From montanaro at users.sourceforge.net Thu Sep 18 10:04:33 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 10:04:54 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.33,1.34
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv30688
Modified Files:
storage.py
Log Message:
* one import per line
* it's not necessary to import sys at the bottom
Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** storage.py 11 Sep 2003 08:34:42 -0000 1.33
--- storage.py 18 Sep 2003 14:04:31 -0000 1.34
***************
*** 63,67 ****
return not not val
! import sys, types
from spambayes import classifier
from spambayes.Options import options
--- 63,68 ----
return not not val
! import sys
! import types
from spambayes import classifier
from spambayes.Options import options
***************
*** 678,681 ****
if __name__ == '__main__':
- import sys
print >> sys.stderr, __doc__
--- 679,681 ----
From montanaro at users.sourceforge.net Thu Sep 18 11:36:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 11:36:35 2003
Subject: [Spambayes-checkins] spambayes/spambayes hammiebulk.py,1.9,1.10
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv17627/spambayes
Modified Files:
hammiebulk.py
Log Message:
Minor performance boost when training on lots of mail. Only display every
tenth message number. More time is spent training and less time waiting
for i/o.
Index: hammiebulk.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/hammiebulk.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -C2 -d -r1.9 -r1.10
*** hammiebulk.py 13 Aug 2003 22:48:38 -0000 1.9
--- hammiebulk.py 18 Sep 2003 15:36:30 -0000 1.10
***************
*** 79,85 ****
for msg in mbox:
i += 1
! sys.stdout.write("\r%6d" % i)
! sys.stdout.flush()
h.train(msg, is_spam)
print
--- 79,88 ----
for msg in mbox:
i += 1
! if i % 10 == 0:
! sys.stdout.write("\r%6d" % i)
! sys.stdout.flush()
h.train(msg, is_spam)
+ sys.stdout.write("\r%6d" % i)
+ sys.stdout.flush()
print
***************
*** 90,96 ****
for msg in mbox:
i += 1
! sys.stdout.write("\r%6d" % i)
! sys.stdout.flush()
h.untrain(msg, is_spam)
print
--- 93,102 ----
for msg in mbox:
i += 1
! if i % 10 == 0:
! sys.stdout.write("\r%6d" % i)
! sys.stdout.flush()
h.untrain(msg, is_spam)
+ sys.stdout.write("\r%6d" % i)
+ sys.stdout.flush()
print
From montanaro at users.sourceforge.net Thu Sep 18 11:36:32 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 11:36:37 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_mboxtrain.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv17627/scripts
Modified Files:
sb_mboxtrain.py
Log Message:
Minor performance boost when training on lots of mail. Only display every
tenth message number. More time is spent training and less time waiting
for i/o.
Index: sb_mboxtrain.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_mboxtrain.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_mboxtrain.py 9 Sep 2003 07:03:54 -0000 1.2
--- sb_mboxtrain.py 18 Sep 2003 15:36:30 -0000 1.3
***************
*** 107,112 ****
continue
counter += 1
! if loud:
! sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(cfn, "rb")
--- 107,112 ----
continue
counter += 1
! if loud and counter % 10 == 0:
! sys.stdout.write("\r%6d" % counter)
sys.stdout.flush()
f = file(cfn, "rb")
***************
*** 128,133 ****
if loud:
! print (" Trained %d out of %d messages " %
! (trained, counter))
def mbox_train(h, path, is_spam, force):
--- 128,134 ----
if loud:
! sys.stdout.write("\r%6d" % counter)
! sys.stdout.write("\r Trained %d out of %d messages\n" %
! (trained, counter))
def mbox_train(h, path, is_spam, force):
***************
*** 152,157 ****
for msg in mbox:
counter += 1
! if loud:
! sys.stdout.write(" %s\r" % counter)
sys.stdout.flush()
if msg_train(h, msg, is_spam, force):
--- 153,158 ----
for msg in mbox:
counter += 1
! if loud and counter % 10 == 0:
! sys.stdout.write("\r%6d" % counter)
sys.stdout.flush()
if msg_train(h, msg, is_spam, force):
***************
*** 181,186 ****
f.close()
if loud:
! print (" Trained %d out of %d messages " %
! (trained, counter))
def mhdir_train(h, path, is_spam, force):
--- 182,188 ----
f.close()
if loud:
! sys.stdout.write("\r%6d" % counter)
! sys.stdout.write("\r Trained %d out of %d messages\n" %
! (trained, counter))
def mhdir_train(h, path, is_spam, force):
***************
*** 199,204 ****
cfn = fn
tfn = os.path.join(path, "spambayes.tmp")
! if loud:
! sys.stdout.write(" %s \r" % fn)
sys.stdout.flush()
f = file(fn, "rb")
--- 201,206 ----
cfn = fn
tfn = os.path.join(path, "spambayes.tmp")
! if loud and counter % 10 == 0:
! sys.stdout.write("\r%6d" % counter)
sys.stdout.flush()
f = file(fn, "rb")
***************
*** 218,223 ****
if loud:
! print (" Trained %d out of %d messages " %
! (trained, counter))
def train(h, path, is_spam, force, trainnew, removetrained):
--- 220,226 ----
if loud:
! sys.stdout.write("\r%6d" % counter)
! sys.stdout.write("\r Trained %d out of %d messages\n" %
! (trained, counter))
def train(h, path, is_spam, force, trainnew, removetrained):
From richie at entrian.com Thu Sep 18 14:17:20 2003
From: richie at entrian.com (Richie Hindle)
Date: Thu Sep 18 15:46:07 2003
Subject: [Spambayes-checkins] spambayes/spambayes storage.py,1.33,1.34
In-Reply-To:
References:
Message-ID:
[Skip]
> * one import per line
I've never understood why the coding standard recommends that - is there a
concrete reason I'm missing, or is it a matter of taste?
--
Richie Hindle
richie@entrian.com
From montanaro at users.sourceforge.net Thu Sep 18 17:00:13 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Thu Sep 18 17:00:15 2003
Subject: [Spambayes-checkins] spambayes/spambayes tokenizer.py,1.14,1.15
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv27208
Modified Files:
tokenizer.py
Log Message:
worm around possible email pkg bug - see code comment for msg thread
Index: tokenizer.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/tokenizer.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -d -r1.14 -r1.15
*** tokenizer.py 5 Sep 2003 01:15:28 -0000 1.14
--- tokenizer.py 18 Sep 2003 21:00:10 -0000 1.15
***************
*** 805,813 ****
yield 'content-disposition:' + x.lower()
! fname = msg.get_filename()
! if fname is not None:
! for x in crack_filename(fname):
! yield 'filename:' + x
!
if 0: # disabled; see comment before function
x = msg.get('content-transfer-encoding')
--- 805,820 ----
yield 'content-disposition:' + x.lower()
! try:
! fname = msg.get_filename()
! if fname is not None:
! for x in crack_filename(fname):
! yield 'filename:' + x
! except TypeError:
! # bug in email pkg? see the thread beginning at
! # http://mail.python.org/pipermail/spambayes/2003-September/008006.html
! # and
! # http://mail.python.org/pipermail/spambayes-dev/2003-September/001177.html
! yield "filename:"
!
if 0: # disabled; see comment before function
x = msg.get('content-transfer-encoding')
From anadelonbrin at users.sourceforge.net Thu Sep 18 18:06:02 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:06:06 2003
Subject: [Spambayes-checkins] spambayes MANIFEST.in,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv8875
Modified Files:
MANIFEST.in
Log Message:
Add extra files needed for pop3proxy_tray.
Index: MANIFEST.in
===================================================================
RCS file: /cvsroot/spambayes/spambayes/MANIFEST.in,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** MANIFEST.in 6 Sep 2003 03:12:46 -0000 1.6
--- MANIFEST.in 18 Sep 2003 22:05:59 -0000 1.7
***************
*** 6,10 ****
recursive-include utilities *.py *.txt
recursive-include testtools *.py *.txt
! recursive-include windows *.py *.txt
include *.txt *.py
--- 6,10 ----
recursive-include utilities *.py *.txt
recursive-include testtools *.py *.txt
! recursive-include windows *.py *.txt *.h *.rc *.ico
include *.txt *.py
From anadelonbrin at users.sourceforge.net Thu Sep 18 18:20:39 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:20:42 2003
Subject: [Spambayes-checkins] spambayes CHANGELOG.txt,1.18,1.19
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv12399
Modified Files:
CHANGELOG.txt
Log Message:
Bring up to date.
Index: CHANGELOG.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/CHANGELOG.txt,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** CHANGELOG.txt 9 Sep 2003 08:21:19 -0000 1.18
--- CHANGELOG.txt 18 Sep 2003 22:20:36 -0000 1.19
***************
*** 3,6 ****
--- 3,41 ----
Alpha Release 6
===============
+ Skip Montanaro 19/09/2003 Worm around a possible email pkg bug.
+ Skip Montanaro 19/09/2003 hammiebulk & mboxtrain: Minor performance boost when training on lots of mail.
+ Skip Montanaro 19/09/2003 Place a threshold on the number of items displayed per section in the review page of the ui.
+ Tony Meyer 18/09/2003 Change the Outlook plug-in to use the general default (currently False) for the experimental_ham_spam_imbalance adjustment.
+ Tony Meyer 18/09/2003 Change the default for address_headers to include to, cc, reply-to, and sender as per Tim's suggestion.
+ Tony Meyer 18/09/2003 Store the user's caches and messageinfo.db file in the app data folder as well (by default), if we have everything else there.
+ Tony Meyer 18/09/2003 If we can't find a config file anywhere but the windows app data directory, then load it.
+ Tony Meyer 18/09/2003 Use urllib not urllib2 to shut down the proxy.
+ Tony Meyer 18/09/2003 Create the server strings for the ui *after* reading in the command line parameters.
+ Tony Meyer 18/09/2003 Move notate_to and notate_subject options to the "Headers" section.
+ Tony Meyer 18/09/2003 Move all the storage options in the "pop3proxy" section to the "Storage" section.
+ Tony Meyer 18/09/2003 Remove the hammie debug options, and use the "Headers" evidence options instead.
+ Tony Meyer 17/09/2003 Fix [ spambayes-Bugs-806632 ] sb_server failure when saving config
+ Tony Meyer 17/09/2003 Outlook: Add a warning for those with highly (>5 times) imbalanced ham and spam.
+ Mark Hammond 17/09/2003 pop3proxy_service: Only munge sys.path in source-code versions, and as the service starts, have it report the username and ini file it is using. In binary builds, write a log to %temp%\SpamBayesServicen.log
+ Mark Hammond 17/09/2003 If we are running Windows, have no valid config file specified, and have the win32all extensions available, default to: \Documents and Settings\[user]\Application Data\SpamBayes\Proxy
+ Tony Meyer 16/09/2003 Fix [ 795145 ] pop3proxy review page dies with mixed gzip/non messages
+ Mark Hammond 15/09/2003 Outlook: Load dialog bitmaps directly from the executable in binary builds.
+ Adam Walker 15/09/2003 Updated pop3proxy_tray to support the service.
+ Tony Meyer 15/09/2003 Removed the gary_combining option and code.
+ Tony Meyer 15/09/2003 Add a (basic) check for version option to the pop3proxy tray app.
+ Tim Peters 15/09/2003 Fix SF bug 806238: urllib2 fails in Outlook new-version chk.
+ Tim Peters 15/09/2003 Outlook: ShowClues(): Made the clue report a little prettier, and (I hope) a little easier to follow.
+ Tony Meyer 13/09/2003 Fix [ 805351 ] If cc: address is not qualified, proxy fails to send message
+ Mark Hammond 12/09/2003 pop3proxy_tray: Use simple logging strategy similar to the Outlook addin - if we have no console, redirect either to win32traceutil (source-code version) or to a %TEMP\SpamBayesServer1.log (yet to be released binary version).
+ Mark Hammond 12/09/2003 pop3proxy_tray: When running from binary, don't hack sys.path. When running from source code, hack sys.path based file path rather than on os.getcwd.
+ Mark Hammond 12/09/2003 pop3proxy_tray: When running from binary, load the icon from the executable rather than a .ico file.
+ Tim Peters 12/09/2003 Outlook: ShowClues(): Add lines revealing the # ham and spam trained on.
+ Tony Meyer 11/09/2003 When running setup.py if the old (named) files exist, offer to delete them.
+ Tony Meyer 11/09/2003 Add a unittest to check that we correctly fail when no dbm modules are available.
+ Anthony Baxter 11/09/2003 Add a new file: NEWTRICKS.TXT to record ideas that have and haven't been tried.
+ Richie Hindle 11/09/2003 Bug 803501: Fix the "No dbm modules available" message to print rather than crash.
+ Skip Montanaro 11/09/2003 Implement a better fix for the storage.py pickle/dbm problems.
+ Mark Hammond 10/09/2003 Outlook: use the classifier's (new) store method rather than an Outlook specific one.
+ Tony Meyer 10/09/2003 Re-fix storage.py so that hammie works with a pickle or dbm.
Tony Meyer 09/09/2003 Fix for [ 802545 ] crash when loggin off imapfilter UI
Tony Meyer 09/09/2003 Fix for [ 802347 ] multiline options saved incorrectly
From anadelonbrin at users.sourceforge.net Thu Sep 18 18:56:11 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:56:19 2003
Subject: [Spambayes-checkins]
spambayes WHAT_IS_NEW.txt, 1.17, 1.18 setup.py, 1.21, 1.22
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv19204
Modified Files:
WHAT_IS_NEW.txt setup.py
Log Message:
Install the chkopts script.
Provide information about what's new in the 1.0a6 release.
Bump up a few version numbers.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -d -r1.17 -r1.18
*** WHAT_IS_NEW.txt 4 Sep 2003 07:14:57 -0000 1.17
--- WHAT_IS_NEW.txt 18 Sep 2003 22:56:06 -0000 1.18
***************
*** 10,14 ****
noted in the "Transition" section.
! New in Alpha Release 5
======================
--- 10,14 ----
noted in the "Transition" section.
! New in Alpha Release 6
======================
***************
*** 17,159 ****
--------------------------
! The values taken by some options have changed, so if you're upgrading from a
! previous version, you may need to update your configuration file (.spambayesrc
! or bayescustomize.ini)
! o allow_remote_connections now takes a list of allowed IP addresses, or the
! word 'localhost', or an asterisk (meaning all connections are accepted).
! o notate_to and notate_subject now take a comma-separated list of one or
! more of 'spam', 'ham' and 'unsure', allowing you to control which classes
! of message are notated.
Outlook Plugin
--------------
! o Added a diagnostics dialog with functions to make it easier for users to
! help developers track down and fix bugs.
! o Added a 'timer' method of determining when to filter mail that should work
! better with Outlook's rule system.
! o Added a button on the Advanced tab of the dialog to display the SpamBayes
! data folder.
! o Moved "Filter Now" to an item on the drop down menu on the toolbar.
! o Items that can be filtered and trained include "IPM.Note" (normal messages)
! and "IPM.Anti-Virus*" (virus alerts by some software).
! o Changed the default filter action to "move" (instead of "untouched").
! o Added a Wizard to assist with initial configuration (this will present
! itself when necessary).
! o Changed to allow filtering to be enabled, even if no training has been done.
! o Added a "New Folder" button to the folder selector dialog.
! o Massive changes to the dialog system (which should fix some problems),
! including changing the configuration dialog to a tabbed interface.
! o "Show Clues" now shows the percentage, as well as the raw score.
! o Added a "Help" menu to the drop down menu, with various information.
! o Added the ability to check for the latest version via an item on the drop
! down menu.
! o Hopefully, the "unread flag" issue is now fixed.
! o Fixed many problems with working on systems where English is not the
! default language, or where profile names have non-English characters.
! POP3 Proxy / SMTP Proxy / POP3 Proxy Service
! --------------------------------------------
! o Fixed "assert hamcount <= nham" problem.
! o Starting and stopping the POP3 Proxy service (for Windows NT, Windows
! 2000 and Windows XP users) has been improved. Most noticeably, this
! means that the SMTP Proxy will start (if it is needed) as well.
! o Improve the "notate to" and "notate subject" options, so that ham and
! unsure messages can also be (optionally) notated in these fields.
! o Add the ability to skip caching messages that are over a (user
! configurable) size, so that you can keep the size of the cache
! directories smaller, once these messages are correctly classified.
! o Added the ability to skip caching messages that have a precedence of
! "bulk" (most mailing list messages), so that you can keep the size
! of the cache directories (and review list) smaller, once these messages
! are correctly classified.
! o Fixed the "ASCII decoding error" problem.
! o The SMTP proxy tries harder to pass on the command formatted exactly
! as it was given. This should make it more reliable.
! o Add the ability to have the SMTP proxy train on the message sent to it,
! rather than looking up the id in the cache (which is still possible, and
! generally the better option).
! o Removed the ability to add the SpamBayes identification number to the
! body of messages (it can still be added as a header).
! o The review messages page now puts unsure messages at the top.
! o The POP3 proxy should now work with fetchmail.
! o You can once again specify local addresses as well as ports for the
! POP3 proxy to listen on (was broken in 1.0a3 and 1.0a4).
! o A bug with the SMTP proxy that would show up in some cases as an
! "unrecognised command" error the mail client (particularly Eudora)
! was fixed.
IMAP Filter
-----------
! o If you didn't use the -p switch to enter your password
! interactively, imapfilter would try and get it from the options,
! but if it wasn't there yet (because you hadn't done the setup yet),
! it would crash. This is now fixed.
General
-------
! o Added the ability to store the SpamBayes database in a mySQL or
! postreSGL database table (currently supported by hammiefilter and
! the POP3 proxy).
! o Removed the ability to use the 'dumbdbm' as the storage method.
! (See the FAQ for reasons why).
! o We now allow the '@' and '=' characters in paths.
! o Added a simple n-way classifier using a cascade of binary SpamBayes
! classifiers.
! o Added version information to the web interface.
! o Fixed the yellow colour of the header boxes in the web interface.
! o Fixed restoring defaults from the web interface.
! o Added a missing line break in the status pane on the web interface
! when there are no proxies configured.
! o Prevent the "Show clues" links on the web interface's training page
! from word-wrapping and making all the table rows two lines high.
! o You can now put "*" at the end of a word in the "Word Query" box
! on the web interface, and have it show you the first ten words,
! and how many words there are in total, in the database that start with
! that word.
! o The web interface now supports HTTP-Auth.
! o Added a new script (code-named 'overkill.py') which enables
! 'drag and drop' training for POP3 users. This is currently still in
! the experimental stage, and anyone interested in trying it out should
! enquire on the SpamBayes mailing list
! ().
Developer
---------
! o Created a directory for test suites, including a storage.py test.
! o An empty 'allowed values' now allows an empty string.
! o Add a get_option method, so an option instance itself can be fetched.
! o Support fetching the "latest" set of version data from the spambayes
! web site.
Transition
==========
! If you are transitioning from a version older than 1.0a4, please also
read the notes in the previous release notes (accessible from
).
! o If you were previously using the 'dumbdbm' storage method (you will
! have files called "hammie.db.dat", "hamie.db.dir" and "hammie.db.bak",
! rather than one file called "hammie.db"), then you will need to
! change to using either a pickle (please see the FAQ:
! ), bsddb, gdbm, or one of the new SQL
! based storage methods. The 'dumbdbm' storage method resulting in
! many databases being corrupted, and was never the best choice for
! storage, in any case. Although you can use the dbExpImp.py script
! to convert your database to your new storage system, we recommend
! that you retrain from scratch, as it is most likely that your
! database has been corrupted.
! o If you were using the options to notate the "To" or "Subject" headers
! with the message's classification, you will need to update your
! configuration file, as the format for these options have changed.
! o The ability to add the SpamBayes id to the message body has been
! removed, which means that Outlook Express users can no longer use
! the SMTP proxy and have it retrieve messages from the cache. These
! users can use the SMTP proxy by training on the forwarded message
! itself, but this is not recommended, as clues in the message will have
! changed (the "From" address will be yours, for example). At this time,
! you will have to use the web interface for training, although there is
! the possibility of 'drag and drop' training being added in a release in
! the near future.
--- 17,138 ----
--------------------------
! There are two major changes in this release, which *will* effect you if you
! are upgrading from an older version:
! o The scripts have all moved (in the archive), and their names have been
! changed. If you run "setup.py install", it will offer to remove the old
! ones for you, which we recommend. In the archive, the scripts are all
! in a 'scripts' directory, and all the scripts start with the "sb_"
! prefix, to avoid clashing with similiarly named scripts from other
! packages. Some name changes go further - "pop3proxy" is now named
! "sb_server", "hammiefilter" is now named "sb_filter", "hammiecli" is now
! named "sb_client", "hammiesrv" is now named "sb_xmlrpcserver", "proxytee"
! is now named "sb_upload", and the experimental "overkill" script is now
! named "sb_pop3dnd".
!
! If you were previously using the "hammie.py" script, you will notice that
! it is no longer available. We recommend that you use either "sb_filter"
! (probably with "sb_mboxtrain"), or use "sb_server" and "sb_upload". If you
! wish to continue as you were, you can use the "hammie.py" module, which
! will be installed in the "spambayes" package directory, in the same way you
! used the old "hammie.py" script.
!
! o All the backwards compatibility code for options which changed names has
! been removed, which means that you *must* use the correct (new) names.
! A script (sb_chkopts) is provided which, if you run it, will inform you
! if you have any invalid names (if will not output anything if there are
! no problems).
!
! In addition, the values taken by some options have changed, so if you're
! upgrading from a previous version, you may need to update your configuration
! file (.spambayesrc or bayescustomize.ini)
!
! o The options to put the classification in the subject or recipient list
! (notate_to and notate_subject) have moved from the "pop3proxy" section
! to the "Headers" section.
! o All the "pop3proxy" storage options (where the cache is stored, the
! number of days before messages expire, and so on) have moved to the
! "Storage" section.
! o The "hammie" debug header options have been removed, and you should use
! the "Headers" evidence header options instead.
!
! Note that pop3proxy (sb_server) and imapfilter users can simply use the web
! interface to check their options and correct any that are wrong. All
! incorrectlly named options in the configuration file will be removed.
Outlook Plugin
--------------
! o Change the default for the ham/spam imbalance adjustment option to
! False - this should make misclassifications for those with large
! imbalances easier to understand. Note that we recommend roughly equal
! numbers of ham and spam are trained.
! o Add a warning for those with highly imbalanced ham and spam.
! o Improved the 'Show Clues' results page.
! o When we fail to add the 'Spam' field to a read-only store (eg, hotmail),
! complain less loudly.
! POP3 Proxy / SMTP Proxy
! -----------------------
! o If running Windows, and have the win32 extensions installed, and a
! configuration file cannot be found, then default to placing it in the
! "Application Data" directory (as defined by Windows). Also default to
! storing the caches and databases in this directory.
! o Correctly save and close the database when changing options. This
! fixes a bug for those using gdbm databases.
!
! Web Interface
! -------------
! o Place a threshold on the number of items displayed per section in the
! review page.
! o Correctly display the proxy data when it is specified on the command
! line.
!
! POP3 Proxy Service / POP3 Proxy Tray Application
! ------------------------------------------------
!
! o The POP3 proxy tray application (for Windows users), which quietly
! appeared in the 1.0a5 release, should be ready for use. If you have
! the service installed, it will attempt to use that; if you do not,
! it will start up sb_server for you.
! o As the service starts, it reports the username and configuration file
! that it is using.
! o Added a version check option to the tray application.
! o Improved logged for both the service and the tray application.
IMAP Filter
-----------
! o Correctly handle IMAP servers that (wrongly) fail to put folder names
! in quotation marks
! o Count all messages being classified instead of just the ones from the
! last folder.
! o Handle a folder name as a literal when presenting a list to choose from.
! o Handle imap servers that do not pass a blank result line for an empty
! search.
! o Fix IMAP over SSL.
General
-------
! o Add a new file: NEWTRICKS.TXT to record ideas that have and haven't
! been tried. If you would like to add to this, please submit a patch
! via the Sourceforge system.
! o Change the default to tokenizer the "To", "CC", "Reply-To", "Sender",
! and "From" address headers, rather than just "From".
! o Fix the database opening code so that hammie* again works with both
! dbm and pickles.
! o Minor performance boosts to hammiebulk and mboxtrain.
Developer
---------
! o Add a close method to the various storage classes.
! o Removed the gary_combining option and code.
Transition
==========
! If you are transitioning from a version older than 1.0a5, please also
read the notes in the previous release notes (accessible from
).
! Other than the changes listed in "Incompatible changes" at the top of
! this document, there are no known transition issues.
***************
*** 161,169 ****
===================
The following bugs tracked via the Sourceforge system were fixed:
! 776808, 795749, 787251, 790051, 743131, 779319, 785389, 786952, 788495,
! 790406, 788008, 787296, 788002, 780612, 784323, 784296, 780819, 780801,
! 779049, 765912, 777026, 777165, 693387, 690418, 719586, 769346, 761499,
! 769346, 773452, 765042, 760062, 768162, 768221, 797776, 797316, 796996,
! 797890, 788845
A url containing the details of these bugs can be made by appending the
--- 140,145 ----
===================
The following bugs tracked via the Sourceforge system were fixed:
! 803501, 802545, 802347, 801952, 798362, 800555, 806632, 795145, 806238,
! 805351
A url containing the details of these bugs can be made by appending the
***************
*** 174,192 ****
Feature Requests Added
======================
! The following feature requests tracked via the Sourceforge system were
! added:
! 789916, 698036, 796832, 791319
!
! A url containing the details of these feature requests can be made by
! appending the request number to this url:
! http://sourceforge.net/tracker/index.php?func=detail&group_id=61702&atid=498106&aid=
Patches integrated
===================
! The following patches tracked via the Sourceforge system were integrated:
! 791254, 790615, 788001, 769981, 791393
!
! A url containing the details of these feature requests can be made by
! appending the request number to this url:
! http://sourceforge.net/tracker/index.php?func=detail&group_id=61702&atid=498105&aid=
--- 150,159 ----
Feature Requests Added
======================
! No feature requests tracked via the Sourceforge system were added for this
! release.
Patches integrated
===================
! No patches tracked via the Sourceforge system were integrated for this
! release.
Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -d -r1.21 -r1.22
*** setup.py 11 Sep 2003 14:22:38 -0000 1.21
--- setup.py 18 Sep 2003 22:56:06 -0000 1.22
***************
*** 86,89 ****
--- 86,90 ----
'scripts/sb_upload.py',
'scripts/sb_xmlrpcserver.py',
+ 'scripts/sb_chkopts.py',
],
packages = [
From anadelonbrin at users.sourceforge.net Thu Sep 18 18:56:13 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 18:56:22 2003
Subject: [Spambayes-checkins] spambayes/spambayes Version.py,1.23,1.24
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv19204/spambayes
Modified Files:
Version.py
Log Message:
Install the chkopts script.
Provide information about what's new in the 1.0a6 release.
Bump up a few version numbers.
Index: Version.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/Version.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** Version.py 18 Sep 2003 03:58:59 -0000 1.23
--- Version.py 18 Sep 2003 22:56:09 -0000 1.24
***************
*** 28,31 ****
--- 28,33 ----
# Sub-dict for application specific version strings.
"Apps": {
+ # Should this be here? I'm not sure that anything uses it, and the
+ # hammie.py script is gone. A sb_filter version might be better.
"Hammie" : {
"Version": 0.1,
***************
*** 48,54 ****
},
"POP3 Proxy" : {
! "Version": 0.2,
"BinaryVersion": 0.1,
! "Description": "SpamBayes POP3 Proxy Beta2",
"Date": "September 2003",
"InterfaceVersion": 0.03,
--- 50,56 ----
},
"POP3 Proxy" : {
! "Version": 0.3,
"BinaryVersion": 0.1,
! "Description": "SpamBayes POP3 Proxy Beta3",
"Date": "September 2003",
"InterfaceVersion": 0.03,
***************
*** 71,82 ****
},
"SMTP Proxy" : {
! "Version": 0.02,
! "Description": "SpamBayes SMTP Proxy Alpha2",
"Date": "September 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
},
"IMAP Filter" : {
! "Version": 0.02,
! "Description": "SpamBayes IMAP Filter Alpha2",
"Date": "September 2003",
"InterfaceVersion": 0.02,
--- 73,84 ----
},
"SMTP Proxy" : {
! "Version": 0.1,
! "Description": "SpamBayes SMTP Proxy Beta1",
"Date": "September 2003",
"Full Description": "%(Description)s, version %(Version)s (%(Date)s)",
},
"IMAP Filter" : {
! "Version": 0.1,
! "Description": "SpamBayes IMAP Filter Beta1",
"Date": "September 2003",
"InterfaceVersion": 0.02,
From mhammond at users.sourceforge.net Thu Sep 18 20:19:24 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Thu Sep 18 20:19:27 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/installer
spambayes_addin.iss, 1.11, 1.12
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/installer
In directory sc8-pr-cvs1:/tmp/cvs-serv1549
Modified Files:
spambayes_addin.iss
Log Message:
Ooops - forgot to checkin the 0.81 change for the setup script.
Index: spambayes_addin.iss
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/installer/spambayes_addin.iss,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** spambayes_addin.iss 8 Sep 2003 08:17:29 -0000 1.11
--- spambayes_addin.iss 19 Sep 2003 00:19:22 -0000 1.12
***************
*** 5,10 ****
[Setup]
AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.8
! AppVersion=0.8
DefaultDirName={pf}\Spambayes Outlook Addin
DefaultGroupName=Spambayes Outlook Addin
--- 5,10 ----
[Setup]
AppName=Spambayes Outlook Addin
! AppVerName=Spambayes Outlook Addin 0.81
! AppVersion=0.81
DefaultDirName={pf}\Spambayes Outlook Addin
DefaultGroupName=Spambayes Outlook Addin
From anadelonbrin at users.sourceforge.net Thu Sep 18 21:04:18 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 21:04:21 2003
Subject: [Spambayes-checkins] spambayes/spambayes __init__.py,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv9208/spambayes
Modified Files:
__init__.py
Log Message:
Release 1.0a6
Index: __init__.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/__init__.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** __init__.py 4 Sep 2003 21:30:46 -0000 1.6
--- __init__.py 19 Sep 2003 01:04:16 -0000 1.7
***************
*** 1,3 ****
# package marker.
! __version__ = '1.0a5'
--- 1,3 ----
# package marker.
! __version__ = '1.0a6'
From anadelonbrin at users.sourceforge.net Thu Sep 18 23:45:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Thu Sep 18 23:45:16 2003
Subject: [Spambayes-checkins] spambayes WHAT_IS_NEW.txt,1.18,1.19
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1019
Modified Files:
WHAT_IS_NEW.txt
Log Message:
Mark tells me that the tray app will fail right at the top on Win9x (and I preusme ME).
Too much hassle to fix it for 1.0a6, so add a note letting people know that this is the case,
and that it will be resolved in the future.
Index: WHAT_IS_NEW.txt
===================================================================
RCS file: /cvsroot/spambayes/spambayes/WHAT_IS_NEW.txt,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** WHAT_IS_NEW.txt 18 Sep 2003 22:56:06 -0000 1.18
--- WHAT_IS_NEW.txt 19 Sep 2003 03:45:10 -0000 1.19
***************
*** 95,98 ****
--- 95,102 ----
the service installed, it will attempt to use that; if you do not,
it will start up sb_server for you.
+ ** Note that the tray application currently only works with Windows
+ NT, Windows 2000 and Windows XP, not Windows 95, 98, or ME. This
+ will be remedied in a future version, and in the binary release of
+ the tray application. **
o As the service starts, it reports the username and configuration file
that it is using.
From mhammond at users.sourceforge.net Fri Sep 19 00:03:40 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 00:03:42 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000 filter.py, 1.32,
1.33 train.py, 1.33, 1.34
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000
In directory sc8-pr-cvs1:/tmp/cvs-serv3727
Modified Files:
filter.py train.py
Log Message:
Remove old dead code from __main__ blocks
Index: filter.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/filter.py,v
retrieving revision 1.32
retrieving revision 1.33
diff -C2 -d -r1.32 -r1.33
*** filter.py 4 Sep 2003 12:14:12 -0000 1.32
--- filter.py 19 Sep 2003 04:03:38 -0000 1.33
***************
*** 143,154 ****
def main():
! import manager
! mgr = manager.GetManager()
!
! import dialogs.FilterDialog
! d = dialogs.FilterDialog.FilterArrivalsDialog(mgr, filterer)
! d.DoModal()
! mgr.Save()
! mgr.Close()
if __name__ == "__main__":
--- 143,147 ----
def main():
! print "Sorry - we don't do anything here any more"
if __name__ == "__main__":
Index: train.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/train.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -C2 -d -r1.33 -r1.34
*** train.py 27 Aug 2003 11:28:47 -0000 1.33
--- train.py 19 Sep 2003 04:03:38 -0000 1.34
***************
*** 187,199 ****
def main():
! import manager
! mgr = manager.GetManager()
!
! import dialogs.TrainingDialog
! d = dialogs.TrainingDialog.TrainingDialog(mgr, trainer)
! d.DoModal()
!
! mgr.Save()
! mgr.Close()
if __name__ == "__main__":
--- 187,191 ----
def main():
! print "Sorry - we don't do anything here any more"
if __name__ == "__main__":
From mhammond at users.sourceforge.net Fri Sep 19 01:42:06 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 01:42:09 2003
Subject: [Spambayes-checkins] spambayes/windows/py2exe setup_all.py,1.6,1.7
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows/py2exe
In directory sc8-pr-cvs1:/tmp/cvs-serv17310/py2exe
Modified Files:
setup_all.py
Log Message:
First version of a combined binary.
Index: setup_all.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/py2exe/setup_all.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** setup_all.py 16 Sep 2003 05:21:03 -0000 1.6
--- setup_all.py 19 Sep 2003 05:42:04 -0000 1.7
***************
*** 89,96 ****
)
! outlook_doc_files = [
["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
]
--- 89,100 ----
)
! outlook_data_files = [
["outlook", [os.path.join(sb_top_dir, r"Outlook2000\about.html")]],
["outlook/docs", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\*.html"))],
["outlook/docs/images", glob.glob(os.path.join(sb_top_dir, r"Outlook2000\docs\images\*.jpg"))],
+ ["outlook", [os.path.join(sb_top_dir, r"Outlook2000\default_bayes_customize.ini")]],
+ ]
+ proxy_data_files = [
+ ["proxy", [os.path.join(sb_top_dir, r"windows\readme_proxy.html")]],
]
***************
*** 111,114 ****
windows=[pop3proxy_tray],
# and the misc data files
! data_files = outlook_doc_files,
)
--- 115,118 ----
windows=[pop3proxy_tray],
# and the misc data files
! data_files = outlook_data_files + proxy_data_files,
)
From mhammond at users.sourceforge.net Fri Sep 19 01:42:06 2003
From: mhammond at users.sourceforge.net (Mark Hammond)
Date: Fri Sep 19 01:42:11 2003
Subject: [Spambayes-checkins] spambayes/windows readme_proxy.html, NONE,
1.1 spambayes.iss, NONE, 1.1
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv17310
Added Files:
readme_proxy.html spambayes.iss
Log Message:
First version of a combined binary.
--- NEW FILE: readme_proxy.html ---
SpamBayes
SpamBayes Server/Proxy Application.
Wel come to SpamBayes
This SpamBayes application sits between your Mail Server (generally at
your ISP) and your Mail program. Your mail program still believes
it is talking to a real mail server, but instead it is talking to
SpamBayes, which acts as a proxy
between the two.
SpamBayes will let all good email through normally. All Spam or
Unsure items are held by the application for your review. You
review all messages and configure SpamBayes itself via your web browser.
If you use Microsoft Outlook, you should probably be using the Outlook
Addin, which was presented as an option at installation time.
Getting Started
You should start the SpamBayes Tray
Icon program, which was installed on your Start menu. It
is recommended
Using the Service
There is a windows service program installed, but it is not configured
by the installation program. If you configure this manually, you
can continue to use the Tray Icon program, as it will control the
service instead of running the server internally as it normally does.
To install the service, perform the following steps:
- Open a command prompt, and change to the \Program Files\SpamBayes\proxy
directory.
- Execute pop3proxy_service.exe
-install
- Select Control
Panel->Administrative Tools->Services, and locate SpamBayes Service in the list.
- Change the properties of the service so it logs on with your user
account, rather than the builtin system account. This will ensure
that SpamBayes uses the same configuration and data files when running
as a service and when running as a normal program.
- If desired, change the properties of the service to start at boot
time.
- Start the service
- Start the SpamBayes Tray Icon
program, and confirm the server is running. Configure and manage
spambayes normally.
--- NEW FILE: spambayes.iss ---
;
; Inno Setup 4.x setup file for the Spambayes Binaries
;
[Setup]
AppName=Spambayes
AppVerName=Spambayes 1.0a6
AppVersion=1.0a6
DefaultDirName={pf}\Spambayes
DefaultGroupName=Spambayes
OutputDir=.
OutputBaseFilename=SpamBayes-Setup
ShowComponentSizes=no
; Note the check for Outlook running has already been done, so no point
; having this file tell them to shutdown outlook!
; Edit file using Windows 'wordpad'
;InfoBeforeFile=installation_notes.rtf
[Files]
Source: "py2exe\dist\lib\*.*"; DestDir: "{app}\lib"; Flags: ignoreversion
Source: "py2exe\dist\outlook\spambayes_addin.dll"; DestDir: "{app}\outlook"; Check: InstallingOutlook; Flags: ignoreversion regserver
Source: "py2exe\dist\outlook\docs\welcome.html"; DestDir: "{app}\outlook\docs"; Check: InstallingOutlook; Flags: isreadme
Source: "py2exe\dist\outlook\*.*"; DestDir: "{app}\outlook"; Check: InstallingOutlook; Flags: ignoreversion recursesubdirs
Source: "py2exe\dist\proxy\*.*"; DestDir: "{app}\proxy"; Check: InstallingProxy; Flags: ignoreversion recursesubdirs
Source: "py2exe\dist\proxy\readme_proxy.html"; DestDir: "{app}\proxy"; Check: InstallingProxy; Flags: isreadme
[Tasks]
Name: startup; Description: "Execute SpamBayes each time Windows starts";
Name: desktop; Description: "Add an icon to the desktop"; Flags: unchecked;
[Run]
FileName:"{app}\proxy\pop3proxy_tray.exe"; Description: "Start the server now"; Flags: postinstall skipifdoesntexist nowait
[Icons]
Name: "{group}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy
Name: "{userdesktop}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy; Tasks: desktop
Name: "{userstartup}\SpamBayes Tray Icon"; Filename: "{app}\proxy\pop3proxy_tray.exe"; Check: InstallingProxy; Tasks: startup
Name: "{group}\About SpamBayes"; Filename: "{app}\proxy\readme_proxy.html"; Check: InstallingProxy;
Name: "{group}\SpamBayes Outlook Addin\About SpamBayes"; Filename: "{app}\outlook\about.html"; Check: InstallingOutlook
Name: "{group}\SpamBayes Outlook Addin\Troubleshooting Guide"; Filename: "{app}\outlook\docs\troubleshooting.html"; Check: InstallingOutlook
[UninstallDelete]
[Code]
var
InstallOutlook, InstallProxy: Boolean;
WarnedNoOutlook, WarnedBoth : Boolean;
function InstallingOutlook() : Boolean;
begin
Result := InstallOutlook;
end;
function InstallingProxy() : Boolean;
begin
Result := InstallProxy;
end;
function IsOutlookInstalled() : Boolean;
begin
Result := RegKeyExists( HKCU, 'Software\Microsoft\Office\Outlook');
end;
function InitializeSetup(): Boolean;
begin
Result := true;
while Result do begin
if not CheckForMutexes('_outlook_mutex_') then
break;
Result := MsgBox(
'You must close Outlook before SpamBayes can be installed.' + #13 + #13 +
'Please close all Outlook Windows (using "File->Exit and Log off"' + #13 +
'if available) and click Retry, or click Cancel to exit the installation.'+ #13 + #13 +
'If this message persists after closing all Outlook windows, you may' + #13 +
'need to log off from Windows, and try again.',
mbConfirmation, MB_RETRYCANCEL) = idRetry;
end;
// default our install type.
if IsOutlookInstalled() then begin
InstallOutlook := True;
InstallProxy := False
end
else begin
InstallOutlook := False;
InstallProxy := True;
end;
end;
// Inno has a pretty primitive "Components/Tasks" concept that
// doesn't quite fit what we want - so we create a custom wizard page.
function PromptApplications( BackClicked: Boolean): Boolean;
var
Next: Boolean;
Prompts, Values: array of String;
begin
// First open the custom wizard page
ScriptDlgPageOpen();
// Set some captions
ScriptDlgPageSetCaption('Select applications to install');
ScriptDlgPageSetSubCaption1('A number of applications are included with this package.');
ScriptDlgPageSetSubCaption2('Select the components you wish to install.');
SetArrayLength(Prompts, 2);
SetArrayLength(Values, 2);
if InstallOutlook then
Prompts[0] := 'Microsoft Outlook Addin (Outlook appears to be installed)'
else
Prompts[0] := 'Microsoft Outlook Addin (Outlook does not appear to be installed)';
Prompts[1] := 'Server/Proxy application, for all other POP based mail clients, including Outlook Express';
while True do begin
if InstallOutlook then Values[0] := '1' else Values[0] := '0';
if InstallProxy then Values[1] := '1' else Values[1] := '0';
Next:= InputOptionArray(Prompts, Values, False, False);
if not Next then Break;
InstallOutlook := (Values[0] = '1');
InstallProxy := (Values[1] = '1');
if InstallOutlook and not IsOutlookInstalled and not WarnedNoOutlook then begin
if MsgBox(
'Outlook appears to not be installed.' + #13 + #13 +
'This addin only works with Microsoft Outlook 2000 and later - it' + #13 +
'does not work with Outlook express.' + #13 + #13 +
'If you know that Outlook is installed, you may with to continue.' + #13 + #13 +
'Would you like to change your selection?',
mbConfirmation, MB_YESNO) = idNo then begin
WarnedNoOutlook := True;
Break; // break check loop
end;
Continue;
end;
if InstallOutlook and InstallProxy and not WarnedBoth then begin
if MsgBox(
'You have selected to install both the Outlook Addin and the Server/Proxy Applications.' + #13 + #13 +
'Unless you regularly use both Outlook and another mailer on the same system' + #13 +
'you do not need both applications.' + #13 + #13 +
'Would you like to change your selection?',
mbConfirmation, MB_YESNO) = idNo then begin
WarnedBoth := True;
Break; // break check loop
end;
Continue
end;
if not InstallOutlook and not InstallProxy then begin
MsgBox('You must select one of the applications', mbError, MB_OK);
Continue;
end
// we got to here, we are OK.
Break;
end
// See NextButtonClick and BackButtonClick: return True if the click should be allowed
if not BackClicked then
Result := Next
else
Result := not Next;
// Close the wizard page. Do a FullRestore only if the click (see above) is not allowed
ScriptDlgPageClose(not Result);
end;
function ScriptDlgPages(CurPage: Integer; BackClicked: Boolean): Boolean;
begin
if (not BackClicked and (CurPage = wpWelcome)) or (BackClicked and (CurPage = wpSelectDir)) then begin
// Insert a custom wizard page between two non custom pages
Result := PromptApplications( BackClicked );
end
else
Result := True;
end;
function NextButtonClick(CurPage: Integer): Boolean;
begin
Result := ScriptDlgPages(CurPage, False);
end;
function BackButtonClick(CurPage: Integer): Boolean;
begin
Result := ScriptDlgPages(CurPage, True);
end;
function SkipCurPage(CurPage: Integer): Boolean;
begin
Result := (CurPage = wpSelectTasks) and (not InstallProxy);
end;
From anadelonbrin at users.sourceforge.net Fri Sep 19 03:18:28 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 03:18:30 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.22,1.23
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv1482
Modified Files:
setup.py
Log Message:
If we are removing scripts with the old names, remove any pyc or pyo variants as well,
but quietly.
Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** setup.py 18 Sep 2003 22:56:06 -0000 1.22
--- setup.py 19 Sep 2003 07:18:26 -0000 1.23
***************
*** 60,63 ****
--- 60,70 ----
try:
os.remove(s)
+ # Also remove .pyc and .pyo, but quietly.
+ pyc = "%sc" % (s,)
+ pyo = "%so" % (s,)
+ if os.path.exists(pyc):
+ os.remove(pyc)
+ if os.path.exists(pyo):
+ os.remove(pyo)
print "Removed", s
except OSError:
From anadelonbrin at users.sourceforge.net Fri Sep 19 03:51:08 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 03:51:11 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_smtpproxy.py,1.2,1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv8412/scripts
Modified Files:
sb_smtpproxy.py
Log Message:
How this slipped past earlier tesing, I have no idea.
Fix an incorrect name.
Index: sb_smtpproxy.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_smtpproxy.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** sb_smtpproxy.py 13 Sep 2003 04:03:41 -0000 1.2
--- sb_smtpproxy.py 19 Sep 2003 07:51:06 -0000 1.3
***************
*** 370,374 ****
def extractSpambayesID(self, data):
! msg = message_from_string(data)
# The nicest MUA is one that forwards the header intact.
--- 370,374 ----
def extractSpambayesID(self, data):
! msg = sbheadermessage_from_string(data)
# The nicest MUA is one that forwards the header intact.
From anadelonbrin at users.sourceforge.net Fri Sep 19 04:10:28 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:10:35 2003
Subject: [Spambayes-checkins] spambayes/windows readme_proxy.html,1.1,1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv13583/windows
Modified Files:
readme_proxy.html
Log Message:
Tidy up the documentation a bit, and correct a few mistakes. It was a nice attempt for
someone that doesn't use the proxy, though .
Note that the image links don't work at the moment. I'm not sure where they should try and
find the images, since they're in the Outlook stuff at the moment.
Index: readme_proxy.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/readme_proxy.html,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** readme_proxy.html 19 Sep 2003 05:42:03 -0000 1.1
--- readme_proxy.html 19 Sep 2003 08:10:25 -0000 1.2
***************
*** 7,38 ****
! SpamBayes Server/Proxy Application.
!
! Wel come to SpamBayes
!
! This SpamBayes application sits between your Mail Server (generally at
! your ISP) and your Mail program. Your mail program still believes
it is talking to a real mail server, but instead it is talking to
SpamBayes, which acts as a proxy
! between the two.
!
! SpamBayes will let all good email through normally. All Spam or
! Unsure items are held by the application for your review. You
! review all messages and configure SpamBayes itself via your web browser.
!
! If you use Microsoft Outlook, you should probably be using the Outlook
! Addin, which was presented as an option at installation time.
!
Getting Started
! You should start the SpamBayes Tray
! Icon program, which was installed on your Start menu. It
! is recommended
!
Using the Service
! There is a windows service program installed, but it is not configured
! by the installation program. If you configure this manually, you
can continue to use the Tray Icon program, as it will control the
! service instead of running the server internally as it normally does.
! To install the service, perform the following steps:
- Open a command prompt, and change to the
!
!
!
! |
!
! |
! |
!
!
!
! SpamBayes Server/Proxy Application
! Welcome to SpamBayes
!
! Note that if you use Microsoft Outlook, you should probably be using the
! Outlook Addin, which was presented as an option at installation time.
!
! This SpamBayes application sits between your mail server (generally at
! your ISP) and your mail program. Your mail program still believes
it is talking to a real mail server, but instead it is talking to
SpamBayes, which acts as a proxy
! between the two.
!
! All your mail arrives as normal in your mail program, but, as it arrives,
! SpamBayes adds a classification to the messages - either ham (good
! mail), spam (bad mail), or unsure (messages that SpamBayes isn't
! certain about). You can set your mail program up to automatically
! filter messages into folders based on this classification - for example you
! might move all spam messages to a folder called "Junk Mail", and all
! unsure messages to a folder called "Mail to review".
!
! In order to classify messages, you have to train SpamBayes. The
! easiest method of doing this is via an interface that SpamBayes presents
! to you via your web browser. SpamBayes will list all the messages that have
! been received, and let you train them as ham or spam. You
! can also configure SpamBayes via this interface in your web browser.
!
Getting Started
! You should start the SpamBayes Tray
! Icon program, which was installed into your Start menu. This is the
! recommended method of operating SpamBayes. If you right-click on this
! icon you will be presented with a number of options to control SpamBayes,
! including an option to configure SpamBayes for your system.
!
Using the Service
! There is a windows service program installed, but it is not configured
! by the installation program. If you configure this manually, you
can continue to use the Tray Icon program, as it will control the
! service instead of running the server internally as it normally does.
! To install the service, perform the following steps:
- Open a command prompt, and change to the SpamBayes Service in the list.
- Change the properties of the service so it logs on with your user
! account, rather than the builtin system account. This will ensure
that SpamBayes uses the same configuration and data files when running
as a service and when running as a normal program.
--- 68,72 ----
style="font-style: italic;">SpamBayes Service
in the list.
- Change the properties of the service so it logs on with your user
! account, rather than the builtin system account. This will ensure
that SpamBayes uses the same configuration and data files when running
as a service and when running as a normal program.
***************
*** 51,56 ****
time.
- Start the service
! - Start the SpamBayes Tray Icon
! program, and confirm the server is running. Configure and manage
spambayes normally.
--- 74,79 ----
time.
Start the service
! Start the SpamBayes Tray Icon
! program, and confirm the server is running. Configure and manage
spambayes normally.
From anadelonbrin at users.sourceforge.net Fri Sep 19 04:24:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:24:17 2003
Subject: [Spambayes-checkins] spambayes/Outlook2000/docs welcome.html, 1.5,
1.6
Message-ID:
Update of /cvsroot/spambayes/spambayes/Outlook2000/docs
In directory sc8-pr-cvs1:/tmp/cvs-serv16508/Outlook2000/docs
Modified Files:
welcome.html
Log Message:
Tidy up the welcome doc a bit (mostly internally, hardly any external change).
Index: welcome.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/Outlook2000/docs/welcome.html,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** welcome.html 10 Sep 2003 07:38:51 -0000 1.5
--- welcome.html 19 Sep 2003 08:24:10 -0000 1.6
***************
*** 12,17 ****
|
! |
|
--- 12,17 ----
|
!
! |
|
***************
*** 19,38 ****
! Welcome to SpamBayes
!
SpamBayes is an Outlook plugin that provides a spam filter based on
statistical analysis of
! your personal mail. Unlike many other spam detection systems,
SpamBayes
actually learns what you consider spam, and continually adapts as both
your regular email and spam patterns change.
When you first start Outlook after SpamBayes has been installed, the
! SpamBayes Installation Wizard will appear. This Wizard will guide
you through the configuration process and allow you to quickly have
! SpamBayes filtering your mail. This document contains additional
information which will help make SpamBayes effective from the first
time you use it.
Please remember that this is free software; please be patient, and
! note that there are plenty of things to improve. There are ways
you can help, even if you aren't a programmer - you could help with
this documentation, or
! Welcome to SpamBayes
SpamBayes is an Outlook plugin that provides a spam filter based on
statistical analysis of
! your personal mail. Unlike many other spam detection systems,
SpamBayes
actually learns what you consider spam, and continually adapts as both
your regular email and spam patterns change.
When you first start Outlook after SpamBayes has been installed, the
! SpamBayes Installation Wizard will appear. This Wizard will guide
you through the configuration process and allow you to quickly have
! SpamBayes filtering your mail. This document contains additional
information which will help make SpamBayes effective from the first
time you use it.
Please remember that this is free software; please be patient, and
! note that there are plenty of things to improve. There are ways
you can help, even if you aren't a programmer - you could help with
this documentation, or main project
! page for information. The list of Frequently Asked Questions
may also answer some of yours.
--- 39,43 ----
any number of other ways - please see the main project
! page for information. The list of Frequently Asked Questions
may also answer some of yours.
***************
*** 49,67 ****
rules, so anything you consider spam will be treated as spam by this
system, even if it does not conform to the traditional definitions of
! spam. This means that SpamBayes requires training before it will be
! effective. There are two ways that this training can be done:
! - Allow SpamBayes to learn as it goes. Initially, SpamBayes
will consider all mail items unsure,
and each item will be used to
! train. In this scenario, SpamBayes will take some time to become
! effective. It will rarely make mistakes, but will continue to be
unsure about items until the training information grows.
- Pre-sorting mail into two folders; one containing only examples
! of good messages, and another containing only examples of spam.
SpamBayes will then process all these messages gathering the clues it
! uses to filter mail.. Depending on how many messages you train
on, SpamBayes will be immediately effective at correctly classifying
your mail.
--- 48,66 ----
rules, so anything you consider spam will be treated as spam by this
system, even if it does not conform to the traditional definitions of
! spam. This means that SpamBayes requires training before it will be
! effective. There are two ways that this training can be done:
! - Allow SpamBayes to learn as it goes. Initially, SpamBayes
will consider all mail items unsure,
and each item will be used to
! train. In this scenario, SpamBayes will take some time to become
! effective. It will rarely make mistakes, but will continue to be
unsure about items until the training information grows.
- Pre-sorting mail into two folders; one containing only examples
! of good messages, and another containing only examples of spam.
SpamBayes will then process all these messages gathering the clues it
! uses to filter mail. Depending on how many messages you train
on, SpamBayes will be immediately effective at correctly classifying
your mail.
***************
*** 70,76 ****
It is important to note that even when SpamBayes has little training
information, it rarely gets things wrong - the worst it generally does
! is to classify a message as unsure.
However, as mentioned, the more training information SpamBayes has, the
! less it is unsure about new messages. See using the plugin below
for more information.
Using the Plugin
--- 69,75 ----
It is important to note that even when SpamBayes has little training
information, it rarely gets things wrong - the worst it generally does
! is to classify a message as unsure.
However, as mentioned, the more training information SpamBayes has, the
! less it is unsure about new messages. See using the plugin below
for more information.
Using the Plugin
***************
*** 80,84 ****
This section describes how the
! plugin operates once it is configured.
You can access the SpamBayes features from the SpamBayes toolbar (shown
to the right), but
--- 79,83 ----
|
This section describes how the
! plugin operates once it is configured.
You can access the SpamBayes features from the SpamBayes toolbar (shown
to the right), but
***************
*** 88,92 ****
|
|
--- 87,91 ----
|
***************
*** 95,105 ****
As messages arrive, they are given a spam score. This score is
the measure of how "spammy" the system has decided a mail is, with 100%
being certain spam, and 0% meaning the message is certainly not
! spam. The SpamBayes addin uses these scores to classify mail into
one of three categories -
certain spam, unsure, and good messages.
Good messages are often
known in the anti-spam community as ham.
--- 94,104 ----
As messages arrive, they are given a spam score. This score is
the measure of how "spammy" the system has decided a mail is, with 100%
being certain spam, and 0% meaning the message is certainly not
! spam. The SpamBayes addin uses these scores to classify mail into
one of three categories -
certain spam, unsure, and good messages.
Good messages are often
known in the anti-spam community as ham.
***************
*** 107,111 ****
they
will remain in your inbox, or be filtered normally by Outlook's builtin
! rules. Mail that is classified as either unsure or certain spam
--- 106,110 ----
they
will remain in your inbox, or be filtered normally by Outlook's builtin
! rules. Mail that is classified as either unsure or certain spam
***************
*** 114,131 ****
style="font-style: italic;">spam score to your Outlook folder
views - but you can do it
! manually by following these instructions.
There are three ways in which the system can get things wrong:
! - A spam stays in your inbox. This is known as a false negative. In this case
you can either drag the message to the Spam folder or click on the Delete as Spam button on the
Outlook
! toolbar. In both cases, the message will be trained as spam and
will be moved to the spam folder.
! - Any message is moved to the unsure folder. In this case,
the system is simply unsure about the message, and moves it to the
! possible spam folder for human review. All unsure messages should
be manually classified; good messages can either be dragged back to the
inbox, or have the Recover from Spam
--- 113,130 ----
style="font-style: italic;">spam score to your Outlook folder
views - but you can do it
! manually by following these instructions.
There are three ways in which the system can get things wrong:
! - A spam stays in your inbox. This is known as a false negative. In this case
you can either drag the message to the Spam folder or click on the Delete as Spam button on the
Outlook
! toolbar. In both cases, the message will be trained as spam and
will be moved to the spam folder.
! - Any message is moved to the unsure folder. In this case,
the system is simply unsure about the message, and moves it to the
! possible spam folder for human review. All unsure messages should
be manually classified; good messages can either be dragged back to the
inbox, or have the Recover from Spam
***************
*** 134,155 ****
Spam folder or have the Delete as
Spam
! toobar button (shown above) clicked. In all cases, the system
will
automatically
be trained appropriately.
- A wanted (ham) message is moved to the Spam folder. This is known
! as a false positive. In
this case you can either drag the message back to the folder from which
it came (generally the inbox), or click on the Recover from Spam button. In
both cases the message will be trained as good, and moved back to the
original folder.
Note that in all cases, as you take corrective action on the mail, the
! system is also trained. This makes it less likely that another
! similar mail will be incorrectly classified in the future.
!
!
--- 133,153 ----
Spam folder or have the Delete as
Spam
! toobar button (shown above) clicked. In all cases, the system
will
automatically
be trained appropriately.
- A wanted (ham) message is moved to the Spam folder. This is known
! as a false positive. In
this case you can either drag the message back to the folder from which
it came (generally the inbox), or click on the Recover from Spam button. In
both cases the message will be trained as good, and moved back to the
original folder.
Note that in all cases, as you take corrective action on the mail, the
! system is also trained. This makes it less likely that another
! similar mail will be incorrectly classified in the future.
!
!
From anadelonbrin at users.sourceforge.net Fri Sep 19 04:24:56 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:24:59 2003
Subject: [Spambayes-checkins]
spambayes/spambayes/test test_sb-server.py, 1.1, 1.2
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv16637/spambayes/test
Modified Files:
test_sb-server.py
Log Message:
Fix the sb_server test so that it again runs, and passes.
Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** test_sb-server.py 5 Sep 2003 01:16:46 -0000 1.1
--- test_sb-server.py 19 Sep 2003 08:24:54 -0000 1.2
***************
*** 1,5 ****
#! /usr/bin/env python
! """Test the pop3proxy is working correctly.
When using the -z command line option, carries out a test that the
--- 1,5 ----
#! /usr/bin/env python
! """Test the POP3 proxy is working correctly.
When using the -z command line option, carries out a test that the
***************
*** 83,92 ****
import getopt
! # a bit of a hack to help those without spambayes on their
! # Python path - stolen from timtest.py
import sys
import os
sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.dirname(os.getcwd()))
from spambayes import Dibbler
--- 83,93 ----
import getopt
! # We need to import sb_server, but it may not be on the PYTHONPATH.
! # Hack around this, so that if we are running in a cvs-like setup
! # everything still works.
import sys
import os
sys.path.insert(-1, os.getcwd())
! sys.path.insert(-1, os.path.join(os.getcwd(), "scripts"))
from spambayes import Dibbler
***************
*** 94,99 ****
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ProxyUI import ProxyUserInterface
! from pop3proxy import BayesProxyListener
! from pop3proxy import state, _recreateState
from spambayes.Options import options
--- 95,100 ----
from spambayes.UserInterface import UserInterfaceServer
from spambayes.ProxyUI import ProxyUserInterface
! from sb_server import BayesProxyListener
! from sb_server import state, _recreateState
from spambayes.Options import options
***************
*** 101,105 ****
# is added to the size of each message.
HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \
! options["Hammie", "header_name"]
class TestListener(Dibbler.Listener):
--- 102,106 ----
# is added to the size of each message.
HEADER_EXAMPLE = '%s: xxxxxxxxxxxxxxxxxxxx\r\n' % \
! options["Headers", "classification_header_name"]
class TestListener(Dibbler.Listener):
***************
*** 300,304 ****
while response.find('\n.\r\n') == -1:
response = response + proxy.recv(1000)
! assert response.find(options["Hammie", "header_name"]) >= 0
# Smoke-test the HTML UI.
--- 301,305 ----
while response.find('\n.\r\n') == -1:
response = response + proxy.recv(1000)
! assert response.find(options["Headers", "classification_header_name"]) >= 0
# Smoke-test the HTML UI.
From anadelonbrin at users.sourceforge.net Fri Sep 19 04:28:09 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 04:28:12 2003
Subject: [Spambayes-checkins]
spambayes/spambayes/test test_sb-server.py, 1.2, 1.3
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv17219/spambayes/test
Modified Files:
test_sb-server.py
Log Message:
Comment typo.
Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** test_sb-server.py 19 Sep 2003 08:24:54 -0000 1.2
--- test_sb-server.py 19 Sep 2003 08:28:07 -0000 1.3
***************
*** 8,12 ****
web ui is present.
! The -t option runs a fake POP3 server on port 8810. This is the
same server that the -z option uses, and may be separately run for
other testing purposes.
--- 8,12 ----
web ui is present.
! The -t option runs a fake POP3 server on port 8110. This is the
same server that the -z option uses, and may be separately run for
other testing purposes.
From montanaro at users.sourceforge.net Fri Sep 19 09:14:23 2003
From: montanaro at users.sourceforge.net (Skip Montanaro)
Date: Fri Sep 19 09:14:30 2003
Subject: [Spambayes-checkins] spambayes setup.py,1.23,1.24
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv11396
Modified Files:
setup.py
Log Message:
Be more vocal about removing pyc's and pyo's. In particular, people may
have already run the earlier version of setup.py which discarded the py's
and left the pyc's and pyo's.
I'm not convinced that any of these scripts would have been imported as
modules from the bin directory (I didn't have any leftover pycs), but I'll
take Richie's word for it. ;-)
Index: setup.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/setup.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** setup.py 19 Sep 2003 07:18:26 -0000 1.23
--- setup.py 19 Sep 2003 13:14:20 -0000 1.24
***************
*** 31,46 ****
class install_scripts(parent):
old_scripts=[
! 'unheader.py',
! 'hammie.py',
! 'hammiecli.py',
! 'hammiesrv.py',
! 'hammiefilter.py',
! 'pop3proxy.py',
! 'smtpproxy.py',
! 'proxytee.py',
! 'dbExpImp.py',
! 'mboxtrain.py',
! 'imapfilter.py',
! 'notesfilter.py',
]
--- 31,46 ----
class install_scripts(parent):
old_scripts=[
! 'unheader',
! 'hammie',
! 'hammiecli',
! 'hammiesrv',
! 'hammiefilter',
! 'pop3proxy',
! 'smtpproxy',
! 'proxytee',
! 'dbExpImp',
! 'mboxtrain',
! 'imapfilter',
! 'notesfilter',
]
***************
*** 49,55 ****
for s in self.old_scripts:
s = os.path.join(self.install_dir, s)
! if os.path.exists(s):
! print >> sys.stderr, "Error: old script", s, "still exists."
! err = True
if err:
print >>sys.stderr, "Do you want to delete these scripts? (y/n)"
--- 49,57 ----
for s in self.old_scripts:
s = os.path.join(self.install_dir, s)
! for e in (".py", ".pyc", ".pyo"):
! if os.path.exists(s+e):
! print >> sys.stderr, "Error: old script", s+e,
! print >> sys.stderr, "still exists."
! err = True
if err:
print >>sys.stderr, "Do you want to delete these scripts? (y/n)"
***************
*** 58,73 ****
for s in self.old_scripts:
s = os.path.join(self.install_dir, s)
! try:
! os.remove(s)
! # Also remove .pyc and .pyo, but quietly.
! pyc = "%sc" % (s,)
! pyo = "%so" % (s,)
! if os.path.exists(pyc):
! os.remove(pyc)
! if os.path.exists(pyo):
! os.remove(pyo)
! print "Removed", s
! except OSError:
! pass
return parent.run(self)
--- 60,69 ----
for s in self.old_scripts:
s = os.path.join(self.install_dir, s)
! for e in (".py", ".pyc", ".pyo"):
! try:
! os.remove(s+e)
! print "Removed", s+e
! except OSError:
! pass
return parent.run(self)
From anadelonbrin at users.sourceforge.net Fri Sep 19 18:47:15 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 18:47:20 2003
Subject: [Spambayes-checkins]
spambayes/spambayes/test test_sb-server.py, 1.3, 1.4
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/test
In directory sc8-pr-cvs1:/tmp/cvs-serv3518/spambayes/test
Modified Files:
test_sb-server.py
Log Message:
Another comment typo.
Index: test_sb-server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/test/test_sb-server.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** test_sb-server.py 19 Sep 2003 08:28:07 -0000 1.3
--- test_sb-server.py 19 Sep 2003 22:47:13 -0000 1.4
***************
*** 14,18 ****
Usage:
! pop3proxytest.py [options]
options:
--- 14,18 ----
Usage:
! test_sb-server.py [options]
options:
From anadelonbrin at users.sourceforge.net Fri Sep 19 18:48:31 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 18:48:34 2003
Subject: [Spambayes-checkins] spambayes/windows pop3proxy_tray.py,1.11,1.12
Message-ID:
Update of /cvsroot/spambayes/spambayes/windows
In directory sc8-pr-cvs1:/tmp/cvs-serv3734/windows
Modified Files:
pop3proxy_tray.py
Log Message:
While explaining how to use the tray app to my fiance, I realised that the most common
operation is probably going to be reviewing messages, so add that as a menu item.
Index: pop3proxy_tray.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/windows/pop3proxy_tray.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -C2 -d -r1.11 -r1.12
*** pop3proxy_tray.py 15 Sep 2003 04:48:50 -0000 1.11
--- pop3proxy_tray.py 19 Sep 2003 22:48:29 -0000 1.12
***************
*** 115,122 ****
self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
1025 : ("-", None),
! 1026 : ("View information ...", self.OpenInterface),
! 1027 : ("Configure ...", self.OpenConfig),
! 1028 : ("Check for latest version", self.CheckVersion),
! 1029 : ("-", None),
1099 : ("Exit SpamBayes", self.OnExit),
}
--- 115,123 ----
self.control_functions = {START_STOP_ID : ("Stop SpamBayes", self.StartStop),
1025 : ("-", None),
! 1026 : ("Review messages ...", self.OpenReview),
! 1027 : ("View information ...", self.OpenInterface),
! 1028 : ("Configure ...", self.OpenConfig),
! 1029 : ("Check for latest version", self.CheckVersion),
! 1030 : ("-", None),
1099 : ("Exit SpamBayes", self.OnExit),
}
***************
*** 410,413 ****
--- 411,421 ----
if self.started:
webbrowser.open_new("http://localhost:%d/config" % \
+ (options["html_ui", "port"],))
+ else:
+ self.ShowMessage("SpamBayes is not running.")
+
+ def OpenReview(self):
+ if self.started:
+ webbrowser.open_new("http://localhost:%d/review" % \
(options["html_ui", "port"],))
else:
From anadelonbrin at users.sourceforge.net Fri Sep 19 19:12:55 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:12:59 2003
Subject: [Spambayes-checkins]
website applications.ht, 1.22, 1.23 download.ht,
1.18, 1.19 index.ht, 1.26, 1.27
Message-ID:
Update of /cvsroot/spambayes/website
In directory sc8-pr-cvs1:/tmp/cvs-serv7774
Modified Files:
applications.ht download.ht index.ht
Log Message:
Release 1.0a6.
Index: applications.ht
===================================================================
RCS file: /cvsroot/spambayes/website/applications.ht,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** applications.ht 8 Sep 2003 10:55:58 -0000 1.22
--- applications.ht 19 Sep 2003 23:12:52 -0000 1.23
***************
*** 11,22 ****
Outlook
Sean True and Mark Hammond have developed an addin for Outlook (2000 and XP) that
! adds support for the spambayes classifier.
Requirements
- Outlook 2000 or Outlook XP (not Outlook Express)
! If you are going to use a source code version, you will also need:
! - Python 2.2 or later (2.2.3 recommended)
- Python's win32com
extensions (win32all-149 or later - currently ActivePython is not suitable)
--- 11,22 ----
Outlook
Sean True and Mark Hammond have developed an addin for Outlook (2000 and XP) that
! adds support for the spambayes classifier.
Requirements
- Outlook 2000 or Outlook XP (not Outlook Express)
! If you are going to use a source code version, you will also need:
! - Python 2.2 or later (2.3 recommended)
- Python's win32com
extensions (win32all-149 or later - currently ActivePython is not suitable)
***************
*** 29,83 ****
Mark has packaged together an installer for the plugin.
Go to the Windows page for more.
! Download the alpha4 release.
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! hammie is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!).
! Currently it focusses on running hammie via procmail.
Requirements
! - Python 2.2 or later (2.2.3 recommended)
- Currently documentation focuses on Unix.
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! pop3proxy sits between your mail client and your real POP3 server and marks
! mail as ham or spam as it passes through. See the docstring at the top of pop3proxy.py for more.
Requirements
! - Python 2.2 or later (2.2.3 recommended)
- Should work on windows/unix/whatever...
-
Availability
! Download the alpha4 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! imap filter connects to your imap server and marks mail as ham or spam,
moving it to appropriate folders as it arrives.
! See the docstring at the top of imapfilter.py for more.
! Note that this application is 'more' alpha than the others.
Requirements
! - Python 2.2 or later (2.2.3 recommended)
- Should work on windows/unix/whatever...
-
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
This application allows you to train incrementally on ham and spam
messages stored in Unix mailbox format files. When run, it adds an extra
--- 29,81 ----
Mark has packaged together an installer for the plugin.
Go to the Windows page for more.
! Download the alpha6 release.
Alternatively, you can use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! sb_filter is a command line tool for marking mail as ham or spam. Skip Montanaro has started a
guide to integrating hammie with your mailer (Unix-only instructions at the moment - additions welcome!).
! Currently it focusses on running sb_filter via procmail.
Requirements
! - Python 2.2 or later (2.3 recommended)
- Currently documentation focuses on Unix.
Availability
! Download the alpha6 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! sb_server provides a POP3 proxy which sits between your mail client and your real POP3 server and marks
! mail as ham or spam as it passes through. See the README for more.
! sb_server can also be used with the sb_upload.py script as a procmail (or similar) solution.
Requirements
! - Python 2.2 or later (2.3 recommended)
- Should work on windows/unix/whatever...
Availability
! Download the alpha6 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
! imapfilter connects to your imap server and marks mail as ham or spam,
moving it to appropriate folders as it arrives.
! See the README for more.
Requirements
! - Python 2.2 or later (2.3 recommended)
- Should work on windows/unix/whatever...
Availability
! Download the alpha6 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
This application allows you to train incrementally on ham and spam
messages stored in Unix mailbox format files. When run, it adds an extra
***************
*** 87,99 ****
Requirements
! - Python 2.2 or later (2.2.3 recommended)
- The code is currently Unix-specific.
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
This application allows you to filter Lotus Notes folders, rather like
the IMAP filter works, moving mail to the correct folder as it arrives.
--- 85,97 ----
Requirements
! - Python 2.2 or later (2.3 recommended)
- The code is currently Unix-specific.
Availability
! Download the alpha6 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
!
This application allows you to filter Lotus Notes folders, rather like
the IMAP filter works, moving mail to the correct folder as it arrives.
***************
*** 105,113 ****
Requirements
! - Python 2.2 or later (2.2.3 recommended)
- Should work on windows/unix/whatever...
Availability
! Download the alpha5 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
--- 103,111 ----
Requirements
! - Python 2.2 or later (2.3 recommended)
- Should work on windows/unix/whatever...
Availability
! Download the alpha6 release.
Alternatively, use CVS to get the code - go to the CVS page on the project's sourceforge site for more.
Index: download.ht
===================================================================
RCS file: /cvsroot/spambayes/website/download.ht,v
retrieving revision 1.18
retrieving revision 1.19
diff -C2 -d -r1.18 -r1.19
*** download.ht 8 Sep 2003 10:55:58 -0000 1.18
--- download.ht 19 Sep 2003 23:12:52 -0000 1.19
***************
*** 4,15 ****
Source Releases
! The fifth pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a5 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
! The primary goal of this pre-release is to shake out any packaging, installation, or integration issues that might be lurking. Feedback to spambayes@python.org.
Prerequisites:
Either:
! - Python 2.2.2, Python 2.3b1, or a CVS build of python, or
- Python 2.2, 2.2.1, plus the latest email package.
--- 4,19 ----
Source Releases
! The sixth (and final alpha) pre-release of version 1.0 of the SpamBayes project is available.
! Download version 1.0a6 from the sourceforge Files page as either a gzipped tarball or a zip file of the source files.
! The primary goal of this pre-release is to shake out any packaging, installation, or integration issues that might be lurking.
! In particular, the script names were changed for this release, and the options code was cleaned up a lot.
! Version 1.0 is now in 'feature freeze', so, while we will continue to resolve any bugs, no new features will be added. This means
! that a beta, and then final, release for version 1.0 is not too far off. New features will be added to the 1.1a1 release.
! Feedback to spambayes@python.org.
Prerequisites:
Either:
! - Python 2.2.2 or above, or a CVS build of python, or
- Python 2.2, 2.2.1, plus the latest email package.
***************
*** 17,21 ****
- if you're running Outlook2000, change to the Outlook2000 directory and run 'addin.py'. See the 'about.html' file in the Outlook2000 directory for more.
!
- otherwise, consult the INTEGRATION.txt file in the package and choose the method that suits your setup best.
--- 21,25 ----
- if you're running Outlook2000, change to the Outlook2000 directory and run 'addin.py'. See the 'about.html' file in the Outlook2000 directory for more.
!
- otherwise, consult the README.txt file in the package and choose the method that suits your setup best.
Index: index.ht
===================================================================
RCS file: /cvsroot/spambayes/website/index.ht,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** index.ht 11 Sep 2003 07:58:34 -0000 1.26
--- index.ht 19 Sep 2003 23:12:52 -0000 1.27
***************
*** 6,10 ****
News
Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
! Fifth pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
--- 6,10 ----
News
Outlook plugin release 008 is available. See the Windows page for more. Note that the Outlook plugin is now available from Sourceforge, on this project's Files page.
! Sixth (and final alpha) pre-release of spambayes available. See the download page for more.
You may also like to see what other people have been saying about us in the press and elsewhere.
From anadelonbrin at users.sourceforge.net Fri Sep 19 19:27:21 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:27:23 2003
Subject: [Spambayes-checkins]
spambayes HAMMIE.txt, 1.5, NONE INTEGRATION.txt, 1.10, NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes
In directory sc8-pr-cvs1:/tmp/cvs-serv10318
Removed Files:
HAMMIE.txt INTEGRATION.txt
Log Message:
All the information in here is now in README.txt, and is more up-to-date.
--- HAMMIE.txt DELETED ---
--- INTEGRATION.txt DELETED ---
From anadelonbrin at users.sourceforge.net Fri Sep 19 19:38:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:38:16 2003
Subject: [Spambayes-checkins] spambayes/scripts sb_server.py, 1.6,
1.7 sb_smtpproxy.py, 1.3, NONE
Message-ID:
Update of /cvsroot/spambayes/spambayes/scripts
In directory sc8-pr-cvs1:/tmp/cvs-serv12134/scripts
Modified Files:
sb_server.py
Removed Files:
sb_smtpproxy.py
Log Message:
If anyone wants to use the smtp proxy, then they can do so via sb_server, with or
without using the pop3 proxy as well. This means that sb_smtpproxy doesn't really
need to exist anymore, and the smtpproxy stuff would be better as a module. Do this.
We had too many message classes! As discussed (by me and Mark, mostly) a long
time back on spambayes-dev, start consolidating these (I was waiting for 1.1).
Add the various interface improvements discussed on spambayes-dev. In particular,
an advanced 'find token' query is available, the 'find message' query is improved,
and the review messages page is more customisable.
Index: sb_server.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/scripts/sb_server.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** sb_server.py 18 Sep 2003 03:58:59 -0000 1.6
--- sb_server.py 19 Sep 2003 23:38:10 -0000 1.7
***************
*** 477,481 ****
# Write the message into the Unknown cache.
message = state.unknownCorpus.makeMessage(msg.getId())
! message.setSubstance(msg.as_string())
state.unknownCorpus.addMessage(message)
--- 477,481 ----
# Write the message into the Unknown cache.
message = state.unknownCorpus.makeMessage(msg.getId())
! message.setPayload(msg.as_string())
state.unknownCorpus.addMessage(message)
***************
*** 493,497 ****
body = re.split(r'\n\r?\n', messageText, 1)[1]
messageText = "\r\n".join(headers) + "\r\n\r\n" + body
-
except:
# Something nasty happened while parsing or classifying -
--- 493,496 ----
***************
*** 751,758 ****
# SMTP proxy information in their configuration, then nothing will
# happen.
! import sb_smtpproxy
! servers, proxyPorts = sb_smtpproxy.LoadServerInfo()
! proxyListeners.extend(sb_smtpproxy.CreateProxies(servers, proxyPorts,
! state))
# setup info for the web interface
--- 750,757 ----
# SMTP proxy information in their configuration, then nothing will
# happen.
! from spambayes import smtpproxy
! servers, proxyPorts = smtpproxy.LoadServerInfo()
! proxyListeners.extend(smtpproxy.CreateProxies(servers, proxyPorts,
! state))
# setup info for the web interface
--- sb_smtpproxy.py DELETED ---
From anadelonbrin at users.sourceforge.net Fri Sep 19 19:38:12 2003
From: anadelonbrin at users.sourceforge.net (Tony Meyer)
Date: Fri Sep 19 19:38:19 2003
Subject: [Spambayes-checkins] spambayes/spambayes/resources ui.html, 1.19,
1.20 ui_html.py, 1.20, 1.21
Message-ID:
Update of /cvsroot/spambayes/spambayes/spambayes/resources
In directory sc8-pr-cvs1:/tmp/cvs-serv12134/spambayes/resources
Modified Files:
ui.html ui_html.py
Log Message:
If anyone wants to use the smtp proxy, then they can do so via sb_server, with or
without using the pop3 proxy as well. This means that sb_smtpproxy doesn't really
need to exist anymore, and the smtpproxy stuff would be better as a module. Do this.
We had too many message classes! As discussed (by me and Mark, mostly) a long
time back on spambayes-dev, start consolidating these (I was waiting for 1.1).
Add the various interface improvements discussed on spambayes-dev. In particular,
an advanced 'find token' query is available, the 'find message' query is improved,
and the review messages page is more customisable.
Index: ui.html
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/resources/ui.html,v
retrieving revision 1.19
retrieving revision 1.20
diff -C2 -d -r1.19 -r1.20
*** ui.html 30 Aug 2003 21:37:10 -0000 1.19
--- ui.html 19 Sep 2003 23:38:10 -0000 1.20
***************
*** 167,171 ****
here, to be trained on later. Click one of the
Discard / Defer / Ham / Spam headers to check all of the
! buttons in that section in one go.