How to determine if a line of python code is a continuation of the line above it

[Leif K-Brooks]

Sandra-24 wrote:
> I'm not sure how complex this is, I've been brainstorming a little, and
> I've come up with:

from tokenize import generate_tokens, NL, NEWLINE
from cStringIO import StringIO

def code_lines(source):
     """Takes Python source code (as either a string or file-like
     object) and yields a tuple of (is_new_logical, code) for each
     physical line of code.
     """

     if isinstance(source, basestring):
         source = StringIO(source)

     buffer = []
     new_logical = True

     for token_type, source, sloc, eloc, line in \
                 generate_tokens(source.readline):
         buffer.append(source)
         if token_type == NL:
             yield new_logical, ''.join(buffer)
             buffer = []
             new_logical = False
         elif token_type == NEWLINE:
             yield new_logical, ''.join(buffer)
             buffer = []
             new_logical = True
     if buffer:
         yield new_logical, ''.join(buffer)