[Python-checkins] r68972 - in doctools/converter/converter: __init__.py latexparser.py restwriter.py tokenizer.py

[georg.brandl]

Author: georg.brandl
Date: Mon Jan 26 22:08:02 2009
New Revision: 68972

Log:
add some support for math.

Modified:
   doctools/converter/converter/__init__.py
   doctools/converter/converter/latexparser.py
   doctools/converter/converter/restwriter.py
   doctools/converter/converter/tokenizer.py

Modified: doctools/converter/converter/__init__.py
==============================================================================
--- doctools/converter/converter/__init__.py	(original)
+++ doctools/converter/converter/__init__.py	Mon Jan 26 22:08:02 2009
@@ -3,7 +3,7 @@
     Documentation converter - high level functions
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: 2007-2008 by Georg Brandl.
+    :copyright: 2007-2009 by Georg Brandl.
     :license: BSD.
 """
 
@@ -42,6 +42,7 @@
                 coutf.close()
         else:
             outf.close()
+        p.finish()  # print warnings about unrecognized commands
         return 1, r.warnings
     except Exception, err:
         if doraise:

Modified: doctools/converter/converter/latexparser.py
==============================================================================
--- doctools/converter/converter/latexparser.py	(original)
+++ doctools/converter/converter/latexparser.py	Mon Jan 26 22:08:02 2009
@@ -5,7 +5,7 @@
 
     For more documentation, look into the ``restwriter.py`` file.
 
-    :copyright: 2007-2008 by Georg Brandl.
+    :copyright: 2007-2009 by Georg Brandl.
     :license: BSD.
 """
 
@@ -84,6 +84,16 @@
     def __init__(self, tokenstream, filename):
         self.tokens = tokenstream
         self.filename = filename
+        self.unrecognized = set()
+
+    def finish(self):
+        if len(self.unrecognized) != 0:
+            print '\n\nWARNING:\nThe following latex commands are not recognized and are ' \
+                'ignored by this script. You may want to extend the DocParser class and ' \
+                'define functions such as handle_CMD to handle them. You can also override '\
+                'handle_unrecognized.\n'
+            for cmd in self.unrecognized:
+                print cmd
 
     def parse(self):
         self.rootnode = RootNode(self.filename, None)
@@ -94,16 +104,25 @@
     def parse_until(self, condition=None, endatbrace=False):
         nodelist = NodeList()
         bracelevel = 0
+        mathmode = False
+        math = ''
         for l, t, v, r in self.tokens:
             if condition and condition(t, v, bracelevel):
                 return nodelist.flatten()
-            if t == 'command':
+            if mathmode:
+                if t == 'mathmode':
+                    nodelist.append(InlineNode('math', [TextNode(math)]))
+                    math = ''
+                    mathmode = False
+                else:
+                    math += r
+            elif t == 'command':
                 if len(v) == 1 and not v.isalpha():
                     nodelist.append(self.handle_special_command(v))
                     continue
                 handler = getattr(self, 'handle_' + v, None)
                 if not handler:
-                    raise ParserError('no handler for \\%s command' % v, l)
+                    handler = self.handle_unrecognized(v, l)
                 nodelist.append(handler())
             elif t == 'bgroup':
                 bracelevel += 1
@@ -116,7 +135,7 @@
             elif t == 'tilde':
                 nodelist.append(NbspNode())
             elif t == 'mathmode':
-                pass # ignore math mode
+                mathmode = True
             elif t == 'parasep':
                 nodelist.append(ParaSepNode())
             else:
@@ -412,6 +431,11 @@
     }
 
     # ------------------------- special handlers -----------------------------
+    def handle_unrecognized(self, name, line):
+        def handler():
+            self.unrecognized.add(name)
+            return EmptyNode()
+        return handler
 
     def handle_special_command(self, cmdname):
         if cmdname in '{}%$^#&_ ':

Modified: doctools/converter/converter/restwriter.py
==============================================================================
--- doctools/converter/converter/restwriter.py	(original)
+++ doctools/converter/converter/restwriter.py	Mon Jan 26 22:08:02 2009
@@ -956,6 +956,9 @@
         # stray commands from distutils
         elif cmdname in ('argument name', 'value', 'attribute', 'option name'):
             self.visit_wrapped('*', content, '*')
+        elif cmdname == 'math':
+            math = node.args[0]
+            self.visit_wrapped(':math:`', math, '`')
         else:
             self.visit_wrapped(':%s:`' % (self.role_mapping[cmdname] or cmdname),
                                self.get_textonly_node(

Modified: doctools/converter/converter/tokenizer.py
==============================================================================
--- doctools/converter/converter/tokenizer.py	(original)
+++ doctools/converter/converter/tokenizer.py	Mon Jan 26 22:08:02 2009
@@ -60,7 +60,7 @@
             elif self.scan(r'\n[ \t]*'):
                 yield lineno, 'text', ' ', self.mtext
                 lineno += 1
-            elif self.scan(r'[^\\%}{\[\]~\n]+'):
+            elif self.scan(r'[^\\%}{\[\]~\n\$]+'):
                 yield lineno, 'text', self.mtext, self.mtext
             else:
                 raise RuntimeError('unexpected text on line %d: %r' %