[Patches] Unicode Patch Set 2000-04-03
M.-A. Lemburg
mal@lemburg.com
Mon, 03 Apr 2000 14:59:18 +0200
This is a multi-part message in MIME format.
--------------F97906822AA778CDBFA10B52
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
The attached patch set adds the following new features:
New Unicode support for int(), float(), complex() and long().
- new APIs PyInt_FromUnicode() and PyLong_FromUnicode()
- added support for Unicode to PyFloat_FromString()
- new encoding API PyUnicode_EncodeDecimal() which converts
Unicode to a decimal char* string (used in the above new
APIs)
- shortcuts for calls like int(<int object>) and float(<float obj>)
- tests for all of the above
Better testing support for the standard codecs.
Misc minor enhancements, such as an alias dbcs for the mbcs codec.
--
Marc-Andre Lemburg
______________________________________________________________________
Business: http://www.lemburg.com/
Python Pages: http://www.lemburg.com/python/
--------------F97906822AA778CDBFA10B52
Content-Type: text/plain; charset=us-ascii;
name="Unicode-Implementation-2000-04-03.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="Unicode-Implementation-2000-04-03.patch"
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Include/Python.h Python+Unicode/Include/Python.h
--- CVS-Python/Include/Python.h Sat Mar 11 10:52:34 2000
+++ Python+Unicode/Include/Python.h Mon Apr 3 13:18:05 2000
@@ -72,6 +72,7 @@
#include "pydebug.h"
+#include "unicodeobject.h"
#include "intobject.h"
#include "longobject.h"
#include "floatobject.h"
@@ -92,7 +93,6 @@
#include "cobject.h"
#include "traceback.h"
#include "sliceobject.h"
-#include "unicodeobject.h"
#include "codecs.h"
#include "pyerrors.h"
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Include/intobject.h Python+Unicode/Include/intobject.h
--- CVS-Python/Include/intobject.h Tue Mar 7 18:58:16 2000
+++ Python+Unicode/Include/intobject.h Mon Apr 3 12:54:35 2000
@@ -60,6 +60,7 @@
#define PyInt_Check(op) ((op)->ob_type == &PyInt_Type)
extern DL_IMPORT(PyObject *) PyInt_FromString Py_PROTO((char*, char**, int));
+extern DL_IMPORT(PyObject *) PyInt_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
extern DL_IMPORT(PyObject *) PyInt_FromLong Py_PROTO((long));
extern DL_IMPORT(long) PyInt_AsLong Py_PROTO((PyObject *));
extern DL_IMPORT(long) PyInt_GetMax Py_PROTO((void));
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Include/longobject.h Python+Unicode/Include/longobject.h
--- CVS-Python/Include/longobject.h Thu Jan 14 11:24:51 1999
+++ Python+Unicode/Include/longobject.h Mon Apr 3 13:00:50 2000
@@ -82,6 +82,7 @@
#endif /* HAVE_LONG_LONG */
DL_IMPORT(PyObject *) PyLong_FromString Py_PROTO((char *, char **, int));
+DL_IMPORT(PyObject *) PyLong_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
#ifdef __cplusplus
}
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Include/unicodeobject.h Python+Unicode/Include/unicodeobject.h
--- CVS-Python/Include/unicodeobject.h Wed Mar 29 11:50:05 2000
+++ Python+Unicode/Include/unicodeobject.h Mon Apr 3 14:17:10 2000
@@ -358,7 +358,7 @@
/* --- UTF-16 Codecs ------------------------------------------------------ */
-/* Decodes length bytes from a UTF-16 encoded buffer string and return
+/* Decodes length bytes from a UTF-16 encoded buffer string and returns
the corresponding Unicode object.
errors (if non-NULL) defines the error handling. It defaults
@@ -397,7 +397,7 @@
);
/* Returns a Python string object holding the UTF-16 encoded value of
- the Unicode data in s.
+ the Unicode data.
If byteorder is not 0, output is written according to the following
byte order:
@@ -586,6 +586,37 @@
);
#endif /* MS_WIN32 */
+
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+/* Takes a Unicode string holding a decimal value and writes it into
+ an output buffer using standard ASCII digit codes.
+
+ The output buffer has to provide at least length+1 bytes of storage
+ area. The output string is 0-terminated.
+
+ The encoder converts whitespace to ' ', decimal characters to their
+ corresponding ASCII digit and all other Latin-1 characters as-is.
+ Characters outside the Latin-1 range (Unicode ordinals 0-256) are
+ treated as errors.
+
+ Error handling is defined by the errors argument:
+
+ NULL or "strict": raise a ValueError
+ "ignore": ignore the wrong characters (these are not copied to the
+ output buffer)
+ "replace": replaces illegal characters with '?'
+
+ Returns 0 on success, -1 on failure.
+
+*/
+
+extern DL_IMPORT(int) PyUnicode_EncodeDecimal(
+ Py_UNICODE *s, /* Unicode buffer */
+ int length, /* Number of Py_UNICODE chars to encode */
+ char *output, /* Output buffer; must have size >= length */
+ const char *errors /* error handling */
+ );
/* --- Methods & Slots ----------------------------------------------------
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Lib/encodings/__init__.py Python+Unicode/Lib/encodings/__init__.py
--- CVS-Python/Lib/encodings/__init__.py Thu Mar 23 23:58:42 2000
+++ Python+Unicode/Lib/encodings/__init__.py Mon Apr 3 13:41:40 2000
@@ -4,8 +4,8 @@
directory.
Codec modules must have names corresponding to standard lower-case
- encoding names. Hyphens are automatically converted to
- underscores, e.g. 'utf-8' is looked up as module utf_8.
+ encoding names with hyphens mapped to underscores, e.g. 'utf-8' is
+ implemented by the module 'utf_8.py'.
Each codec module must export the following interface:
@@ -40,7 +40,7 @@
return entry
# Import the module
- modname = string.replace(encoding,'-','_')
+ modname = string.replace(encoding, '-', '_')
modname = aliases.aliases.get(modname,modname)
try:
mod = __import__(modname,globals(),locals(),'*')
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Lib/encodings/aliases.py Python+Unicode/Lib/encodings/aliases.py
--- CVS-Python/Lib/encodings/aliases.py Mon Apr 3 13:45:48 2000
+++ Python+Unicode/Lib/encodings/aliases.py Sat Apr 1 20:14:36 2000
@@ -54,4 +54,7 @@
'macroman': 'mac_roman',
'macturkish': 'mac_turkish',
+ # MBCS
+ 'dbcs': 'mbcs',
+
}
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Lib/test/output/test_unicode Python+Unicode/Lib/test/output/test_unicode
--- CVS-Python/Lib/test/output/test_unicode Wed Mar 29 11:50:15 2000
+++ Python+Unicode/Lib/test/output/test_unicode Mon Apr 3 14:46:51 2000
@@ -3,3 +3,4 @@
Testing Unicode contains method... done.
Testing Unicode formatting strings... done.
Testing builtin codecs... done.
+Testing standard mapping codecs... 0-127... 128-255... done.
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Lib/test/test_b1.py Python+Unicode/Lib/test/test_b1.py
--- CVS-Python/Lib/test/test_b1.py Fri Mar 26 15:58:20 1999
+++ Python+Unicode/Lib/test/test_b1.py Mon Apr 3 13:35:19 2000
@@ -95,6 +95,7 @@
if complex(0j, 3.14) <> 3.14j: raise TestFailed, 'complex(0j, 3.14)'
if complex(0.0, 3.14) <> 3.14j: raise TestFailed, 'complex(0.0, 3.14)'
if complex(" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(" 3.14+J )"'
+if complex(u" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(u" 3.14+J )"'
class Z:
def __complex__(self): return 3.14j
z = Z()
@@ -208,6 +209,9 @@
if float(314) <> 314.0: raise TestFailed, 'float(314)'
if float(314L) <> 314.0: raise TestFailed, 'float(314L)'
if float(" 3.14 ") <> 3.14: raise TestFailed, 'float(" 3.14 ")'
+if float(u" 3.14 ") <> 3.14: raise TestFailed, 'float(u" 3.14 ")'
+if float(u" \u0663.\u0661\u0664 ") <> 3.14:
+ raise TestFailed, 'float(u" \u0663.\u0661\u0664 ")'
print 'getattr'
import sys
@@ -270,6 +274,19 @@
('', ValueError),
(' ', ValueError),
(' \t\t ', ValueError),
+ (u'0', 0),
+ (u'1', 1),
+ (u'9', 9),
+ (u'10', 10),
+ (u'99', 99),
+ (u'100', 100),
+ (u'314', 314),
+ (u' 314', 314),
+ (u'\u0663\u0661\u0664 ', 314),
+ (u' \t\t 314 \t\t ', 314),
+ (u'', ValueError),
+ (u' ', ValueError),
+ (u' \t\t ', ValueError),
]
for s, v in L:
for sign in "", "+", "-":
@@ -353,6 +370,8 @@
LL = [
('1' + '0'*20, 10L**20),
('1' + '0'*100, 10L**100),
+ (u'1' + u'0'*20, 10L**20),
+ (u'1' + u'0'*100, 10L**100),
]
for s, v in L + LL:
for sign in "", "+", "-":
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Lib/test/test_unicode.py Python+Unicode/Lib/test/test_unicode.py
--- CVS-Python/Lib/test/test_unicode.py Wed Mar 29 11:50:14 2000
+++ Python+Unicode/Lib/test/test_unicode.py Sat Apr 1 00:05:35 2000
@@ -270,11 +270,88 @@
assert unicode(u.encode(encoding),encoding) == u
u = u''.join(map(unichr, range(256)))
-for encoding in ('latin-1',):
+for encoding in (
+ 'latin-1',
+ ):
+ try:
assert unicode(u.encode(encoding),encoding) == u
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
u = u''.join(map(unichr, range(128)))
-for encoding in ('ascii',):
+for encoding in (
+ 'ascii',
+ ):
+ try:
assert unicode(u.encode(encoding),encoding) == u
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
+
+print 'done.'
+
+print 'Testing standard mapping codecs...',
+
+print '0-127...',
+s = ''.join(map(chr, range(128)))
+for encoding in (
+ 'cp037', 'cp1026',
+ 'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
+ 'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
+ 'cp863', 'cp865', 'cp866',
+ 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+ 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
+ 'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
+ 'mac_cyrillic', 'mac_latin2',
+
+ 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+ 'cp1256', 'cp1257', 'cp1258',
+ 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+
+ 'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+ 'cp1006', 'cp875', 'iso8859_8',
+
+ ### These have undefined mappings:
+ #'cp424',
+
+ ):
+ try:
+ assert unicode(s,encoding).encode(encoding) == s
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
+
+print '128-255...',
+s = ''.join(map(chr, range(128,256)))
+for encoding in (
+ 'cp037', 'cp1026',
+ 'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
+ 'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
+ 'cp863', 'cp865', 'cp866',
+ 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+ 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
+ 'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
+ 'mac_cyrillic', 'mac_latin2',
+
+ ### These have undefined mappings:
+ #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+ #'cp1256', 'cp1257', 'cp1258',
+ #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+ #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+
+ ### These fail the round-trip:
+ #'cp1006', 'cp875', 'iso8859_8',
+
+ ):
+ try:
+ assert unicode(s,encoding).encode(encoding) == s
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
print 'done.'
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Misc/unicode.txt Python+Unicode/Misc/unicode.txt
--- CVS-Python/Misc/unicode.txt Wed Mar 29 11:50:15 2000
+++ Python+Unicode/Misc/unicode.txt Mon Apr 3 14:46:51 2000
@@ -1,5 +1,5 @@
=============================================================================
- Python Unicode Integration Proposal Version: 1.2
+ Python Unicode Integration Proposal Version: 1.3
-----------------------------------------------------------------------------
@@ -203,8 +203,9 @@
codecs.register(search_function)
Search functions are expected to take one argument, the encoding name
-in all lower case letters, and return a tuple of functions (encoder,
-decoder, stream_reader, stream_writer) taking the following arguments:
+in all lower case letters and with hyphens and spaces converted to
+underscores, and return a tuple of functions (encoder, decoder,
+stream_reader, stream_writer) taking the following arguments:
encoder and decoder:
These must be functions or methods which have the same
Only in CVS-Python/Objects: .#stringobject.c.2.59
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Objects/abstract.c Python+Unicode/Objects/abstract.c
--- CVS-Python/Objects/abstract.c Sat Mar 11 10:55:08 2000
+++ Python+Unicode/Objects/abstract.c Mon Apr 3 14:40:06 2000
@@ -736,8 +736,16 @@
if (o == NULL)
return null_error();
+ if (PyInt_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (PyString_Check(o))
return PyInt_FromString(PyString_AS_STRING(o), NULL, 10);
+ if (PyUnicode_Check(o))
+ return PyInt_FromUnicode(PyUnicode_AS_UNICODE(o),
+ PyUnicode_GET_SIZE(o),
+ 10);
m = o->ob_type->tp_as_number;
if (m && m->nb_int)
return m->nb_int(o);
@@ -814,6 +822,10 @@
if (o == NULL)
return null_error();
+ if (PyLong_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (PyString_Check(o))
/* need to do extra error checking that PyLong_FromString()
* doesn't do. In particular long('9.5') must raise an
@@ -821,6 +833,11 @@
*/
return long_from_string(PyString_AS_STRING(o),
PyString_GET_SIZE(o));
+ if (PyUnicode_Check(o))
+ /* The above check is done in PyLong_FromUnicode(). */
+ return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),
+ PyUnicode_GET_SIZE(o),
+ 10);
m = o->ob_type->tp_as_number;
if (m && m->nb_long)
return m->nb_long(o);
@@ -838,6 +855,10 @@
if (o == NULL)
return null_error();
+ if (PyFloat_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (!PyString_Check(o)) {
m = o->ob_type->tp_as_number;
if (m && m->nb_float)
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Objects/floatobject.c Python+Unicode/Objects/floatobject.c
--- CVS-Python/Objects/floatobject.c Sat Mar 11 10:55:08 2000
+++ Python+Unicode/Objects/floatobject.c Mon Apr 3 14:42:03 2000
@@ -163,6 +163,22 @@
if (PyString_Check(v)) {
s = PyString_AS_STRING(v);
len = PyString_GET_SIZE(v);
+ }
+ else if (PyUnicode_Check(v)) {
+ char s_buffer[256];
+
+ if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "float() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ s_buffer,
+ NULL))
+ return NULL;
+ s = s_buffer;
+ len = strlen(s);
}
else if (PyObject_AsCharBuffer(v, &s, &len)) {
PyErr_SetString(PyExc_TypeError,
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Objects/intobject.c Python+Unicode/Objects/intobject.c
--- CVS-Python/Objects/intobject.c Sat Mar 11 10:55:08 2000
+++ Python+Unicode/Objects/intobject.c Mon Apr 3 14:42:35 2000
@@ -259,6 +259,24 @@
if (pend)
*pend = end;
return PyInt_FromLong(x);
+}
+
+PyObject *
+PyInt_FromUnicode(s, length, base)
+ Py_UNICODE *s;
+ int length;
+ int base;
+{
+ char buffer[256];
+
+ if (length >= sizeof(buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "int() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(s, length, buffer, NULL))
+ return NULL;
+ return PyInt_FromString(buffer, NULL, base);
}
/* Methods */
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Objects/longobject.c Python+Unicode/Objects/longobject.c
--- CVS-Python/Objects/longobject.c Wed Jan 19 18:01:30 2000
+++ Python+Unicode/Objects/longobject.c Mon Apr 3 14:43:08 2000
@@ -783,6 +783,57 @@
if (pend)
*pend = str;
return (PyObject *) z;
+}
+
+PyObject *
+PyLong_FromUnicode(u, length, base)
+ Py_UNICODE *u;
+ int length;
+ int base;
+{
+ char s_buffer[256];
+ char *s;
+ const char *start;
+ char *end;
+ PyObject *x;
+ char buffer[256]; /* For errors */
+
+ /* Convert to decimal string */
+ if (length >= sizeof(s_buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "long() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(u, length, s_buffer, NULL))
+ return NULL;
+ s = s_buffer;
+
+ /* Convert to long object and apply the same error handling
+ as is done for PyInt_FromString(). */
+ start = s;
+ while (*s && isspace(Py_CHARMASK(*s)))
+ s++;
+ x = PyLong_FromString((char*)s, &end, base);
+ if (x == NULL) {
+ if (PyErr_ExceptionMatches(PyExc_ValueError))
+ goto bad;
+ return NULL;
+ }
+ while (*end && isspace(Py_CHARMASK(*end)))
+ end++;
+ if (*end != '\0') {
+ bad:
+ sprintf(buffer, "invalid literal for long(): %.200s", s);
+ PyErr_SetString(PyExc_ValueError, buffer);
+ Py_XDECREF(x);
+ return NULL;
+ }
+ else if (end != start + length) {
+ PyErr_SetString(PyExc_ValueError,
+ "null byte in argument for long()");
+ return NULL;
+ }
+ return x;
}
static PyLongObject *x_divrem
Only in CVS-Python/Objects: stringobject.c.orig
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Objects/unicodeobject.c Python+Unicode/Objects/unicodeobject.c
--- CVS-Python/Objects/unicodeobject.c Mon Apr 3 13:46:02 2000
+++ Python+Unicode/Objects/unicodeobject.c Mon Apr 3 14:43:52 2000
@@ -1923,6 +1923,60 @@
return NULL;
}
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+int PyUnicode_EncodeDecimal(Py_UNICODE *s,
+ int length,
+ char *output,
+ const char *errors)
+{
+ Py_UNICODE *p, *end;
+
+ if (output == NULL) {
+ PyErr_BadArgument();
+ return -1;
+ }
+
+ p = s;
+ end = s + length;
+ while (p < end) {
+ register Py_UNICODE ch = *p++;
+ int decimal;
+
+ if (Py_UNICODE_ISSPACE(ch)) {
+ *output++ = ' ';
+ continue;
+ }
+ decimal = Py_UNICODE_TODECIMAL(ch);
+ if (decimal >= 0) {
+ *output++ = '0' + decimal;
+ continue;
+ }
+ if (ch < 256) {
+ *output++ = ch;
+ continue;
+ }
+ /* All other characters are considered invalid */
+ if (errors == NULL || strcmp(errors, "strict") == 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "invalid numeric string");
+ goto onError;
+ }
+ else if (strcmp(errors, "ignore") == 0)
+ continue;
+ else if (strcmp(errors, "replace") == 0) {
+ *output++ = '?';
+ continue;
+ }
+ }
+ /* 0-terminate the output string */
+ *output++ = '\0';
+ return 0;
+
+ onError:
+ return -1;
+}
+
/* --- Helpers ------------------------------------------------------------ */
static
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Python/bltinmodule.c Python+Unicode/Python/bltinmodule.c
--- CVS-Python/Python/bltinmodule.c Sat Mar 11 10:55:20 2000
+++ Python+Unicode/Python/bltinmodule.c Mon Apr 3 14:41:16 2000
@@ -449,17 +449,44 @@
PyObject *v;
{
extern double strtod Py_PROTO((const char *, char **));
- char *s, *start, *end;
+ const char *s, *start;
+ char *end;
double x=0.0, y=0.0, z;
int got_re=0, got_im=0, done=0;
int digit_or_dot;
int sw_error=0;
int sign;
char buffer[256]; /* For errors */
+ int len;
- start = s = PyString_AS_STRING(v);
+ if (PyString_Check(v)) {
+ s = PyString_AS_STRING(v);
+ len = PyString_GET_SIZE(v);
+ }
+ else if (PyUnicode_Check(v)) {
+ char s_buffer[256];
+
+ if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "complex() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ s_buffer,
+ NULL))
+ return NULL;
+ s = s_buffer;
+ len = strlen(s);
+ }
+ else if (PyObject_AsCharBuffer(v, &s, &len)) {
+ PyErr_SetString(PyExc_TypeError,
+ "complex() needs a string first argument");
+ return NULL;
+ }
/* position on first nonblank */
+ start = s;
while (*s && isspace(Py_CHARMASK(*s)))
s++;
if (s[0] == '\0') {
@@ -475,7 +502,7 @@
switch (*s) {
case '\0':
- if (s-start != PyString_GET_SIZE(v)) {
+ if (s-start != len) {
PyErr_SetString(
PyExc_ValueError,
"null byte in argument for complex()");
@@ -584,7 +611,7 @@
i = NULL;
if (!PyArg_ParseTuple(args, "O|O:complex", &r, &i))
return NULL;
- if (PyString_Check(r))
+ if (PyString_Check(r) || PyUnicode_Check(r))
return complex_from_string(r);
if ((nbr = r->ob_type->tp_as_number) == NULL ||
nbr->nb_float == NULL ||
@@ -1289,12 +1316,17 @@
return NULL;
if (base == -909)
return PyNumber_Int(v);
- else if (!PyString_Check(v)) {
+ else if (PyString_Check(v))
+ return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
+ else if (PyUnicode_Check(v))
+ return PyInt_FromUnicode(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ base);
+ else {
PyErr_SetString(PyExc_TypeError,
"can't convert non-string with explicit base");
return NULL;
}
- return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
}
static char int_doc[] =
diff -u -b -rP -x *.o -x *.pyc -x Makefile -x *~ -x *.so -x add2lib -x pgen -x buildno -x config.* -x libpython* -x python -x Setup -x Setup.local -x Setup.thread -x hassignal -x Makefile.pre -x *.bak -x *.s -x DEADJOE -x Demo -x CVS -x Doc CVS-Python/Python/codecs.c Python+Unicode/Python/codecs.c
--- CVS-Python/Python/codecs.c Mon Apr 3 13:46:03 2000
+++ Python+Unicode/Python/codecs.c Sat Apr 1 19:57:52 2000
@@ -84,8 +84,11 @@
return -1;
}
+/* Convert a string to a normalized Python string: all characters are
+ converted to lower case, spaces are replaced with underscores. */
+
static
-PyObject *lowercasestring(const char *string)
+PyObject *normalizestring(const char *string)
{
register int i;
int len = strlen(string);
@@ -96,8 +99,14 @@
if (v == NULL)
return NULL;
p = PyString_AS_STRING(v);
- for (i = 0; i < len; i++)
- p[i] = tolower(string[i]);
+ for (i = 0; i < len; i++) {
+ register char ch = string[i];
+ if (ch == ' ')
+ ch = '-';
+ else
+ ch = tolower(ch);
+ p[i] = ch;
+ }
return v;
}
@@ -132,8 +141,10 @@
goto onError;
}
- /* Convert the encoding to a lower-cased Python string */
- v = lowercasestring(encoding);
+ /* Convert the encoding to a normalized Python string: all
+ characters are converted to lower case, spaces and hypens are
+ replaced with underscores. */
+ v = normalizestring(encoding);
if (v == NULL)
goto onError;
PyString_InternInPlace(&v);
--------------F97906822AA778CDBFA10B52--