Another 2 to 3 mail encoding problem

[Chris Green]

I'm unearthing a few issues here trying to convert my mail filter and
delivery programs from 2 to 3!  

I have a simple[ish] local mbox mail delivery module as follows:-


    import mailbox
    import logging
    import logging.handlers
    import os
    import time
    #
    #
    # Class derived from mailbox.mbox so we can override _pre_message_hook()
    # to do nothing instead of appending a blank line
    #
    class mymbox(mailbox.mbox):
        def _pre_message_hook(self, f):
            """Don't write the blank line before the 'From '"""
            pass
    #
    #
    # log a message
    #
    def initLog(name):
        log = logging.getLogger(name)
        log.setLevel(logging.DEBUG)
        f = logging.handlers.RotatingFileHandler("/home/chris/tmp/mail.log", 'a', 1000000, 4)
        f.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        f.setFormatter(formatter)
        log.addHandler(f)
        return log
    #
    #
    # Deliver a message to a local mbox
    #
    def deliverMboxMsg(dest, msg, log):
        #
        #
        # Create the destination mbox instance
        #
        mbx = mymbox(dest, factory=None)

        log.info("From: " + msg.get("From", "unknown"))
        log.info("Destination is: " + dest)
        #
        #
        # Lock the mbox while we append to it
        #
        for tries in range(3):
            try:
                mbx.lock()
                #
                #
                # Append the incoming message to the appropriate mbox
                #
                mbx.add(msg)
                #
                #
                # now set the modified time later than the access time (which is 'now') so
                # that mutt will see new mail in the mbox.
                #
                os.utime(dest, ((time.time()), (time.time() + 5)))
                mbx.unlock()
                break

            except mailbox.ExternalClashError:
                log.info("Destination locked, try " + str(tries))
                time.sleep(1)

        else: # get here if we ran out of tries
            log.warn("Failed to lock destination after 3 attempts, giving up")

        return


It has run faultlessly for many years under Python 2.  I've now
changed the calling program to Python 3 and while it handles most
E-Mail OK I have just got the following error:-

    Traceback (most recent call last):
      File "/home/chris/.mutt/bin/filter.py", line 102, in <module>
        mailLib.deliverMboxMsg(dest, msg, log)
      File "/home/chris/.mutt/bin/mailLib.py", line 52, in deliverMboxMsg
        mbx.add(msg)
      File "/usr/lib/python3.8/mailbox.py", line 603, in add
        self._toc[self._next_key] = self._append_message(message)
      File "/usr/lib/python3.8/mailbox.py", line 758, in _append_message
        offsets = self._install_message(message)
      File "/usr/lib/python3.8/mailbox.py", line 830, in _install_message
        self._dump_message(message, self._file, self._mangle_from_)
      File "/usr/lib/python3.8/mailbox.py", line 215, in _dump_message
        gen.flatten(message)
      File "/usr/lib/python3.8/email/generator.py", line 116, in flatten
        self._write(msg)
      File "/usr/lib/python3.8/email/generator.py", line 181, in _write
        self._dispatch(msg)
      File "/usr/lib/python3.8/email/generator.py", line 214, in _dispatch
        meth(msg)
      File "/usr/lib/python3.8/email/generator.py", line 432, in _handle_text
        super(BytesGenerator,self)._handle_text(msg)
      File "/usr/lib/python3.8/email/generator.py", line 249, in _handle_text
        self._write_lines(payload)
      File "/usr/lib/python3.8/email/generator.py", line 155, in _write_lines
        self.write(line)
      File "/usr/lib/python3.8/email/generator.py", line 406, in write
        self._fp.write(s.encode('ascii', 'surrogateescape'))
    UnicodeEncodeError: 'ascii' codec can't encode character '\ufeff' in position 4: ordinal not in range(128)

So what do I need to do to the message I'm adding with mbx.add(msg) to
fix this?  (I assume that's what I need to do).

-- 
Chris Green
·