line to argv transformation

[Tim Chase]

On 2014-06-16 13:51, Antoon Pardon wrote:
> >>> shlex.split("ls *.py")  
> ['ls', '*.py']
> >>> shlex.split("ls '*.py'")  
> ['ls', '*.py']

To accommodate this, I'd probably just clone the shlib.py to my local
project under a new name and then tweak the source to emit whether a
token was quoted or not, something like the diff below.

You can then iterate over your string/token-stream and know whether
it was quoted or not, allowing you to do any post-processing/globbing
on that file.

-tkc

--- /usr/lib/python2.7/shlex.py	2014-03-13 05:54:53.000000000 -0500
+++ /home/tim/tmp/myshlex.py	2014-06-16 07:39:34.130645270 -0500
@@ -93,29 +93,30 @@
                 print "shlex: popping token " + repr(tok)
             return tok
         # No pushback.  Get a token.
-        raw = self.read_token()
+        was_quoted, raw = self.read_token()
         # Handle inclusions
         if self.source is not None:
             while raw == self.source:
-                spec = self.sourcehook(self.read_token())
+                was_quoted, token = self.read_token()
+                spec = self.sourcehook(roken)
                 if spec:
                     (newfile, newstream) = spec
                     self.push_source(newstream, newfile)
-                raw = self.get_token()
+                was_quoted, raw = self.get_token()
         # Maybe we got EOF instead?
         while raw == self.eof:
             if not self.filestack:
                 return self.eof
             else:
                 self.pop_source()
-                raw = self.get_token()
+                was_quoted, raw = self.get_token()
         # Neither inclusion nor EOF
         if self.debug >= 1:
             if raw != self.eof:
                 print "shlex: token=" + repr(raw)
             else:
                 print "shlex: token=EOF"
-        return raw
+        return was_quoted, raw
 
     def read_token(self):
         quoted = False
@@ -243,7 +244,7 @@
                 print "shlex: raw token=" + repr(result)
             else:
                 print "shlex: raw token=EOF"
-        return result
+        return quoted, result
 
     def sourcehook(self, newfile):
         "Hook called on a filename to be sourced."
@@ -266,10 +267,10 @@
         return self
 
     def next(self):
-        token = self.get_token()
+        was_quoted, token = self.get_token()
         if token == self.eof:
             raise StopIteration
-        return token
+        return was_quoted, token
 
 def split(s, comments=False, posix=True):
     lex = shlex(s, posix=posix)
@@ -285,7 +286,7 @@
         file = sys.argv[1]
         lexer = shlex(open(file), file)
     while 1:
-        tt = lexer.get_token()
+        was_quoted, tt = lexer.get_token()
         if tt:
             print "Token: " + repr(tt)
         else: