[pypy-commit] pypy py3.6: Add an option to allow underscores in integer literals. RPython part.

[amauryfa]

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.6
Changeset: r93621:23240e4a895f
Date: 2018-01-03 22:51 +0100
http://bitbucket.org/pypy/pypy/changeset/23240e4a895f/

Log:	Add an option to allow underscores in integer literals. RPython
	part.

diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -845,7 +845,7 @@
 # String parsing support
 # ---------------------------
 
-def string_to_int(s, base=10):
+def string_to_int(s, base=10, allow_underscores=False):
     """Utility to converts a string to an integer.
     If base is 0, the proper base is guessed based on the leading
     characters of 's'.  Raises ParseStringError in case of error.
@@ -854,7 +854,8 @@
     from rpython.rlib.rstring import (
         NumberStringParser, ParseStringOverflowError, strip_spaces)
     s = literal = strip_spaces(s)
-    p = NumberStringParser(s, literal, base, 'int')
+    p = NumberStringParser(s, literal, base, 'int',
+                           allow_underscores=allow_underscores)
     base = p.base
     result = 0
     while True:
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -272,7 +272,7 @@
 
     @staticmethod
     @jit.elidable
-    def fromstr(s, base=0):
+    def fromstr(s, base=0, allow_underscores=False):
         """As string_to_int(), but ignores an optional 'l' or 'L' suffix
         and returns an rbigint."""
         from rpython.rlib.rstring import NumberStringParser, \
@@ -281,7 +281,8 @@
         if (s.endswith('l') or s.endswith('L')) and base < 22:
             # in base 22 and above, 'L' is a valid digit!  try: long('L',22)
             s = s[:-1]
-        parser = NumberStringParser(s, literal, base, 'long')
+        parser = NumberStringParser(s, literal, base, 'long',
+                                    allow_underscores=allow_underscores)
         return rbigint._from_numberstring_parser(parser)
 
     @staticmethod
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -417,6 +417,9 @@
     def __init__(self, msg):
         self.msg = msg
 
+    def __str__(self):
+        return self.msg
+
 class InvalidBaseError(ParseStringError):
     """Signals an invalid base argument"""
 
@@ -431,7 +434,7 @@
         raise ParseStringError("invalid literal for %s() with base %d" %
                                (self.fname, self.original_base))
 
-    def __init__(self, s, literal, base, fname):
+    def __init__(self, s, literal, base, fname, allow_underscores=False):
         self.fname = fname
         sign = 1
         if s.startswith('-'):
@@ -441,6 +444,8 @@
             s = strip_spaces(s[1:])
         self.sign = sign
         self.original_base = base
+        self.allow_underscores = allow_underscores
+        self.last_is_underscore = False
 
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
@@ -480,13 +485,22 @@
                 digit = (digit - ord('A')) + 10
             elif 'a' <= c <= 'z':
                 digit = (digit - ord('a')) + 10
+            elif c == '_' and self.allow_underscores:
+                if self.last_is_underscore:
+                    self.error()
+                self.last_is_underscore = True
+                self.i += 1
+                return self.next_digit()
             else:
                 self.error()
             if digit >= self.base:
                 self.error()
             self.i += 1
+            self.last_is_underscore = False
             return digit
         else:
+            if self.last_is_underscore:
+                self.error()
             return -1
 
     def prev_digit(self):
diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py
--- a/rpython/rlib/test/test_rarithmetic.py
+++ b/rpython/rlib/test/test_rarithmetic.py
@@ -553,6 +553,51 @@
             py.test.raises(ParseStringError, string_to_int, '+'+s, base)
             py.test.raises(ParseStringError, string_to_int, '-'+s, base)
 
+    def test_number_underscores(self):
+        VALID_UNDERSCORE_LITERALS = [
+            '0_0_0',
+            '4_2',
+            '1_0000_0000',
+            '0b1001_0100',
+            '0xffff_ffff',
+            '0o5_7_7',
+            '0b_0',
+            '0x_f',
+            '0o_5',
+        ]
+        INVALID_UNDERSCORE_LITERALS = [
+            # Trailing underscores:
+            '0_',
+            '42_',
+            '1.4j_',
+            '0x_',
+            '0b1_',
+            '0xf_',
+            '0o5_',
+            # Underscores in the base selector:
+            '0_b0',
+            '0_xf',
+            '0_o5',
+            # Old-style octal, still disallowed:
+            '09_99',
+            # Multiple consecutive underscores:
+            '4_______2',
+            '0b1001__0100',
+            '0xffff__ffff',
+            '0x___',
+            '0o5__77',
+            '1e1__0',
+        ]
+        for x in VALID_UNDERSCORE_LITERALS:
+            print x
+            y = string_to_int(x, base=0, allow_underscores=True)
+            assert y == int(x.replace('_', ''), base=0)
+        for x in INVALID_UNDERSCORE_LITERALS:
+            print x
+            py.test.raises(ParseStringError, string_to_int, x, base=0,
+                           allow_underscores=True)
+
+
 class TestExplicitIntsizes:
 
     _32_max =            2147483647