[Python-checkins] r79725 - in python/trunk: Makefile.pre.in Parser/tokenizer.c
benjamin.peterson
python-checkins at python.org
Sun Apr 4 01:03:36 CEST 2010
Author: benjamin.peterson
Date: Sun Apr 4 01:03:35 2010
New Revision: 79725
Log:
use our own locale independent ctype macros
requires building pyctype.o into pgen
Modified:
python/trunk/Makefile.pre.in
python/trunk/Parser/tokenizer.c
Modified: python/trunk/Makefile.pre.in
==============================================================================
--- python/trunk/Makefile.pre.in (original)
+++ python/trunk/Makefile.pre.in Sun Apr 4 01:03:35 2010
@@ -230,6 +230,7 @@
PGOBJS= \
Objects/obmalloc.o \
Python/mysnprintf.o \
+ Python/pyctype.o \
Parser/tokenizer_pgen.o \
Parser/printgrammar.o \
Parser/pgenmain.o
Modified: python/trunk/Parser/tokenizer.c
==============================================================================
--- python/trunk/Parser/tokenizer.c (original)
+++ python/trunk/Parser/tokenizer.c Sun Apr 4 01:03:35 2010
@@ -92,22 +92,6 @@
"<N_TOKENS>"
};
-
-/* Ensure that the locale does not interfere with tokenization. */
-
-static int
-ascii_isalpha(int c)
-{
- return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
-}
-
-static int
-ascii_isalnum(int c)
-{
- return ascii_isalpha(c) || ('0' <= c && c <= '9');
-}
-
-
/* Create and initialize a new tok_state structure */
static struct tok_state *
@@ -245,7 +229,7 @@
} while (t[0] == '\x20' || t[0] == '\t');
begin = t;
- while (ascii_isalnum(Py_CHARMASK(t[0])) ||
+ while (Py_ISALNUM(t[0]) ||
t[0] == '-' || t[0] == '_' || t[0] == '.')
t++;
@@ -1355,7 +1339,7 @@
}
/* Identifier (most frequent token!) */
- if (ascii_isalpha(c) || c == '_') {
+ if (Py_ISALPHA(c) || c == '_') {
/* Process r"", u"" and ur"" */
switch (c) {
case 'b':
@@ -1381,7 +1365,7 @@
goto letter_quote;
break;
}
- while (ascii_isalnum(c) || c == '_') {
+ while (Py_ISALNUM(c) || c == '_') {
c = tok_nextc(tok);
}
tok_backup(tok, c);
More information about the Python-checkins
mailing list