[Jython-checkins] jython: unicode_literals support.

[frank.wierzbicki]

http://hg.python.org/jython/rev/01869de23bfd
changeset:   6309:01869de23bfd
user:        Frank Wierzbicki <fwierzbicki at gmail.com>
date:        Tue Mar 06 20:07:15 2012 -0800
summary:
  unicode_literals support.

files:
  grammar/Python.g                         |  14 ++++++++---
  src/org/python/antlr/GrammarActions.java |   9 ++++---
  src/org/python/compiler/Future.java      |   2 +
  src/org/python/core/CodeFlag.java        |   6 ++++-
  src/org/python/core/FutureFeature.java   |   4 +++
  5 files changed, 26 insertions(+), 9 deletions(-)


diff --git a/grammar/Python.g b/grammar/Python.g
--- a/grammar/Python.g
+++ b/grammar/Python.g
@@ -166,6 +166,7 @@
     private String encoding;
 
     private boolean printStatement = true;
+    private boolean unicodeLiterals = false;
 
     public void setErrorHandler(ErrorHandler eh) {
         this.errorHandler = eh;
@@ -935,9 +936,14 @@
              if (dottedText != null && dottedText.equals("__future__")) {
                  List<alias> aliases = $i1.atypes;
                  for(alias a: aliases) {
-                     if (a != null && a.getInternalName().equals("print_function")) {
-                         printStatement = false;
-                         break;
+                     if (a != null) {
+                         if (a.getInternalName().equals("print_function")) {
+                             printStatement = false;
+                             break;
+                         } else if (a.getInternalName().equals("unicode_literals")) {
+                             unicodeLiterals = true;
+                             break;
+                         }
                      }
                  }
              }
@@ -1736,7 +1742,7 @@
        }
      | (S+=STRING)+
        {
-           etype = new Str(actions.extractStringToken($S), actions.extractStrings($S, encoding));
+           etype = new Str(actions.extractStringToken($S), actions.extractStrings($S, encoding, unicodeLiterals));
        }
      ;
 
diff --git a/src/org/python/antlr/GrammarActions.java b/src/org/python/antlr/GrammarActions.java
--- a/src/org/python/antlr/GrammarActions.java
+++ b/src/org/python/antlr/GrammarActions.java
@@ -429,14 +429,14 @@
         }
     }
 
-    PyString extractStrings(List s, String encoding) {
+    PyString extractStrings(List s, String encoding, boolean unicodeLiterals) {
         boolean ustring = false;
         Token last = null;
         StringBuffer sb = new StringBuffer();
         Iterator iter = s.iterator();
         while (iter.hasNext()) {
             last = (Token)iter.next();
-            StringPair sp = extractString(last, encoding);
+            StringPair sp = extractString(last, encoding, unicodeLiterals);
             if (sp.isUnicode()) {
                 ustring = true;
             }
@@ -448,12 +448,12 @@
         return new PyString(sb.toString());
     }
 
-    StringPair extractString(Token t, String encoding) {
+    StringPair extractString(Token t, String encoding, boolean unicodeLiterals) {
         String string = t.getText();
         char quoteChar = string.charAt(0);
         int start = 0;
         int end;
-        boolean ustring = false;
+        boolean ustring = unicodeLiterals;
 
         if (quoteChar == 'u' || quoteChar == 'U') {
             ustring = true;
@@ -462,6 +462,7 @@
         if (quoteChar == 'b' || quoteChar == 'B') {
             // In 2.x this is just a str, and the parser prevents a 'u' and a
             // 'b' in the same identifier, so just advance start.
+            ustring = false;
             start++;
         }
         quoteChar = string.charAt(start);
diff --git a/src/org/python/compiler/Future.java b/src/org/python/compiler/Future.java
--- a/src/org/python/compiler/Future.java
+++ b/src/org/python/compiler/Future.java
@@ -62,6 +62,8 @@
                 FutureFeature.absolute_import.addTo(features);
             if (cflags.isFlagSet(CodeFlag.CO_FUTURE_PRINT_FUNCTION))
                 FutureFeature.print_function.addTo(features);
+            if (cflags.isFlagSet(CodeFlag.CO_FUTURE_UNICODE_LITERALS))
+                FutureFeature.unicode_literals.addTo(features);
         }
         int beg = 0;
         List<stmt> suite = null;
diff --git a/src/org/python/core/CodeFlag.java b/src/org/python/core/CodeFlag.java
--- a/src/org/python/core/CodeFlag.java
+++ b/src/org/python/core/CodeFlag.java
@@ -54,7 +54,11 @@
     /**
      * print function.
      */
-    CO_FUTURE_PRINT_FUNCTION(0x10000);
+    CO_FUTURE_PRINT_FUNCTION(0x10000),
+    /**
+     * unicode literals.
+     */
+    CO_FUTURE_UNICODE_LITERALS(0x20000);
 
 
     public final int flag;
diff --git a/src/org/python/core/FutureFeature.java b/src/org/python/core/FutureFeature.java
--- a/src/org/python/core/FutureFeature.java
+++ b/src/org/python/core/FutureFeature.java
@@ -28,6 +28,10 @@
      */
     print_function(CodeFlag.CO_FUTURE_PRINT_FUNCTION),
     /**
+     * Enables unicode literals.
+     */
+    unicode_literals(CodeFlag.CO_FUTURE_UNICODE_LITERALS),
+    /**
      * Use braces for block delimiters instead of indentation.
      */
     braces {

-- 
Repository URL: http://hg.python.org/jython