[Python-checkins] bpo-35134: Create Include/cpython/unicodeobject.h (GH-10680)
Victor Stinner
webhook-mailer at python.org
Mon Nov 26 11:29:41 EST 2018
https://github.com/python/cpython/commit/75e4699b31d1d88abad097ad13466c5c07711324
commit: 75e4699b31d1d88abad097ad13466c5c07711324
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2018-11-26T17:29:38+01:00
summary:
bpo-35134: Create Include/cpython/unicodeobject.h (GH-10680)
Move unicodeobject.h code surrounded by "#ifndef Py_LIMITED_API"
to a new Include/cpython/unicodeobject.h header file.
files:
A Include/cpython/unicodeobject.h
M Include/unicodeobject.h
diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
new file mode 100644
index 000000000000..c76349022485
--- /dev/null
+++ b/Include/cpython/unicodeobject.h
@@ -0,0 +1,1245 @@
+#ifndef Py_CPYTHON_UNICODEOBJECT_H
+# error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Py_UNICODE was the native Unicode storage format (code unit) used by
+ Python and represents a single Unicode element in the Unicode type.
+ With PEP 393, Py_UNICODE is deprecated and replaced with a
+ typedef to wchar_t. */
+#define PY_UNICODE_TYPE wchar_t
+typedef wchar_t Py_UNICODE /* Py_DEPRECATED(3.3) */;
+
+/* --- Internal Unicode Operations ---------------------------------------- */
+
+/* Since splitting on whitespace is an important use case, and
+ whitespace in most situations is solely ASCII whitespace, we
+ optimize for the common case by using a quick look-up table
+ _Py_ascii_whitespace (see below) with an inlined check.
+
+ */
+#define Py_UNICODE_ISSPACE(ch) \
+ ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
+
+#define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
+#define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
+#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
+#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
+
+#define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
+#define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
+#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
+
+#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
+#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
+#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
+#define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch)
+
+#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
+#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
+#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
+
+#define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
+
+#define Py_UNICODE_ISALNUM(ch) \
+ (Py_UNICODE_ISALPHA(ch) || \
+ Py_UNICODE_ISDECIMAL(ch) || \
+ Py_UNICODE_ISDIGIT(ch) || \
+ Py_UNICODE_ISNUMERIC(ch))
+
+#define Py_UNICODE_COPY(target, source, length) \
+ memcpy((target), (source), (length)*sizeof(Py_UNICODE))
+
+#define Py_UNICODE_FILL(target, value, length) \
+ do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
+ for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
+ } while (0)
+
+/* macros to work with surrogates */
+#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF)
+#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF)
+#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF)
+/* Join two surrogate characters and return a single Py_UCS4 value. */
+#define Py_UNICODE_JOIN_SURROGATES(high, low) \
+ (((((Py_UCS4)(high) & 0x03FF) << 10) | \
+ ((Py_UCS4)(low) & 0x03FF)) + 0x10000)
+/* high surrogate = top 10 bits added to D800 */
+#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10))
+/* low surrogate = bottom 10 bits added to DC00 */
+#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF))
+
+/* Check if substring matches at given offset. The offset must be
+ valid, and the substring must not be empty. */
+
+#define Py_UNICODE_MATCH(string, offset, substring) \
+ ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \
+ ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \
+ !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE)))
+
+/* --- Unicode Type ------------------------------------------------------- */
+
+/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
+ structure. state.ascii and state.compact are set, and the data
+ immediately follow the structure. utf8_length and wstr_length can be found
+ in the length field; the utf8 pointer is equal to the data pointer. */
+typedef struct {
+ /* There are 4 forms of Unicode strings:
+
+ - compact ascii:
+
+ * structure = PyASCIIObject
+ * test: PyUnicode_IS_COMPACT_ASCII(op)
+ * kind = PyUnicode_1BYTE_KIND
+ * compact = 1
+ * ascii = 1
+ * ready = 1
+ * (length is the length of the utf8 and wstr strings)
+ * (data starts just after the structure)
+ * (since ASCII is decoded from UTF-8, the utf8 string are the data)
+
+ - compact:
+
+ * structure = PyCompactUnicodeObject
+ * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op)
+ * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+ PyUnicode_4BYTE_KIND
+ * compact = 1
+ * ready = 1
+ * ascii = 0
+ * utf8 is not shared with data
+ * utf8_length = 0 if utf8 is NULL
+ * wstr is shared with data and wstr_length=length
+ if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+ or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4
+ * wstr_length = 0 if wstr is NULL
+ * (data starts just after the structure)
+
+ - legacy string, not ready:
+
+ * structure = PyUnicodeObject
+ * test: kind == PyUnicode_WCHAR_KIND
+ * length = 0 (use wstr_length)
+ * hash = -1
+ * kind = PyUnicode_WCHAR_KIND
+ * compact = 0
+ * ascii = 0
+ * ready = 0
+ * interned = SSTATE_NOT_INTERNED
+ * wstr is not NULL
+ * data.any is NULL
+ * utf8 is NULL
+ * utf8_length = 0
+
+ - legacy string, ready:
+
+ * structure = PyUnicodeObject structure
+ * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND
+ * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
+ PyUnicode_4BYTE_KIND
+ * compact = 0
+ * ready = 1
+ * data.any is not NULL
+ * utf8 is shared and utf8_length = length with data.any if ascii = 1
+ * utf8_length = 0 if utf8 is NULL
+ * wstr is shared with data.any and wstr_length = length
+ if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
+ or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4
+ * wstr_length = 0 if wstr is NULL
+
+ Compact strings use only one memory block (structure + characters),
+ whereas legacy strings use one block for the structure and one block
+ for characters.
+
+ Legacy strings are created by PyUnicode_FromUnicode() and
+ PyUnicode_FromStringAndSize(NULL, size) functions. They become ready
+ when PyUnicode_READY() is called.
+
+ See also _PyUnicode_CheckConsistency().
+ */
+ PyObject_HEAD
+ Py_ssize_t length; /* Number of code points in the string */
+ Py_hash_t hash; /* Hash value; -1 if not set */
+ struct {
+ /*
+ SSTATE_NOT_INTERNED (0)
+ SSTATE_INTERNED_MORTAL (1)
+ SSTATE_INTERNED_IMMORTAL (2)
+
+ If interned != SSTATE_NOT_INTERNED, the two references from the
+ dictionary to this object are *not* counted in ob_refcnt.
+ */
+ unsigned int interned:2;
+ /* Character size:
+
+ - PyUnicode_WCHAR_KIND (0):
+
+ * character type = wchar_t (16 or 32 bits, depending on the
+ platform)
+
+ - PyUnicode_1BYTE_KIND (1):
+
+ * character type = Py_UCS1 (8 bits, unsigned)
+ * all characters are in the range U+0000-U+00FF (latin1)
+ * if ascii is set, all characters are in the range U+0000-U+007F
+ (ASCII), otherwise at least one character is in the range
+ U+0080-U+00FF
+
+ - PyUnicode_2BYTE_KIND (2):
+
+ * character type = Py_UCS2 (16 bits, unsigned)
+ * all characters are in the range U+0000-U+FFFF (BMP)
+ * at least one character is in the range U+0100-U+FFFF
+
+ - PyUnicode_4BYTE_KIND (4):
+
+ * character type = Py_UCS4 (32 bits, unsigned)
+ * all characters are in the range U+0000-U+10FFFF
+ * at least one character is in the range U+10000-U+10FFFF
+ */
+ unsigned int kind:3;
+ /* Compact is with respect to the allocation scheme. Compact unicode
+ objects only require one memory block while non-compact objects use
+ one block for the PyUnicodeObject struct and another for its data
+ buffer. */
+ unsigned int compact:1;
+ /* The string only contains characters in the range U+0000-U+007F (ASCII)
+ and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
+ set, use the PyASCIIObject structure. */
+ unsigned int ascii:1;
+ /* The ready flag indicates whether the object layout is initialized
+ completely. This means that this is either a compact object, or
+ the data pointer is filled out. The bit is redundant, and helps
+ to minimize the test in PyUnicode_IS_READY(). */
+ unsigned int ready:1;
+ /* Padding to ensure that PyUnicode_DATA() is always aligned to
+ 4 bytes (see issue #19537 on m68k). */
+ unsigned int :24;
+ } state;
+ wchar_t *wstr; /* wchar_t representation (null-terminated) */
+} PyASCIIObject;
+
+/* Non-ASCII strings allocated through PyUnicode_New use the
+ PyCompactUnicodeObject structure. state.compact is set, and the data
+ immediately follow the structure. */
+typedef struct {
+ PyASCIIObject _base;
+ Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the
+ * terminating \0. */
+ char *utf8; /* UTF-8 representation (null-terminated) */
+ Py_ssize_t wstr_length; /* Number of code points in wstr, possible
+ * surrogates count as two code points. */
+} PyCompactUnicodeObject;
+
+/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the
+ PyUnicodeObject structure. The actual string data is initially in the wstr
+ block, and copied into the data block using _PyUnicode_Ready. */
+typedef struct {
+ PyCompactUnicodeObject _base;
+ union {
+ void *any;
+ Py_UCS1 *latin1;
+ Py_UCS2 *ucs2;
+ Py_UCS4 *ucs4;
+ } data; /* Canonical, smallest-form Unicode buffer */
+} PyUnicodeObject;
+
+/* Fast access macros */
+#define PyUnicode_WSTR_LENGTH(op) \
+ (PyUnicode_IS_COMPACT_ASCII(op) ? \
+ ((PyASCIIObject*)op)->length : \
+ ((PyCompactUnicodeObject*)op)->wstr_length)
+
+/* Returns the deprecated Py_UNICODE representation's size in code units
+ (this includes surrogate pairs as 2 units).
+ If the Py_UNICODE representation is not available, it will be computed
+ on request. Use PyUnicode_GET_LENGTH() for the length in code points. */
+
+#define PyUnicode_GET_SIZE(op) \
+ (assert(PyUnicode_Check(op)), \
+ (((PyASCIIObject *)(op))->wstr) ? \
+ PyUnicode_WSTR_LENGTH(op) : \
+ ((void)PyUnicode_AsUnicode(_PyObject_CAST(op)),\
+ assert(((PyASCIIObject *)(op))->wstr), \
+ PyUnicode_WSTR_LENGTH(op)))
+ /* Py_DEPRECATED(3.3) */
+
+#define PyUnicode_GET_DATA_SIZE(op) \
+ (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE)
+ /* Py_DEPRECATED(3.3) */
+
+/* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE
+ representation on demand. Using this macro is very inefficient now,
+ try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
+ use PyUnicode_WRITE() and PyUnicode_READ(). */
+
+#define PyUnicode_AS_UNICODE(op) \
+ (assert(PyUnicode_Check(op)), \
+ (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \
+ PyUnicode_AsUnicode(_PyObject_CAST(op)))
+ /* Py_DEPRECATED(3.3) */
+
+#define PyUnicode_AS_DATA(op) \
+ ((const char *)(PyUnicode_AS_UNICODE(op)))
+ /* Py_DEPRECATED(3.3) */
+
+
+/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
+
+/* Values for PyASCIIObject.state: */
+
+/* Interning state. */
+#define SSTATE_NOT_INTERNED 0
+#define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+
+/* Return true if the string contains only ASCII characters, or 0 if not. The
+ string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be
+ ready. */
+#define PyUnicode_IS_ASCII(op) \
+ (assert(PyUnicode_Check(op)), \
+ assert(PyUnicode_IS_READY(op)), \
+ ((PyASCIIObject*)op)->state.ascii)
+
+/* Return true if the string is compact or 0 if not.
+ No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT(op) \
+ (((PyASCIIObject*)(op))->state.compact)
+
+/* Return true if the string is a compact ASCII string (use PyASCIIObject
+ structure), or 0 if not. No type checks or Ready calls are performed. */
+#define PyUnicode_IS_COMPACT_ASCII(op) \
+ (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op))
+
+enum PyUnicode_Kind {
+/* String contains only wstr byte characters. This is only possible
+ when the string was created with a legacy API and _PyUnicode_Ready()
+ has not been called yet. */
+ PyUnicode_WCHAR_KIND = 0,
+/* Return values of the PyUnicode_KIND() macro: */
+ PyUnicode_1BYTE_KIND = 1,
+ PyUnicode_2BYTE_KIND = 2,
+ PyUnicode_4BYTE_KIND = 4
+};
+
+/* Return pointers to the canonical representation cast to unsigned char,
+ Py_UCS2, or Py_UCS4 for direct character access.
+ No checks are performed, use PyUnicode_KIND() before to ensure
+ these will work correctly. */
+
+#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op))
+#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op))
+#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op))
+
+/* Return one of the PyUnicode_*_KIND values defined above. */
+#define PyUnicode_KIND(op) \
+ (assert(PyUnicode_Check(op)), \
+ assert(PyUnicode_IS_READY(op)), \
+ ((PyASCIIObject *)(op))->state.kind)
+
+/* Return a void pointer to the raw unicode buffer. */
+#define _PyUnicode_COMPACT_DATA(op) \
+ (PyUnicode_IS_ASCII(op) ? \
+ ((void*)((PyASCIIObject*)(op) + 1)) : \
+ ((void*)((PyCompactUnicodeObject*)(op) + 1)))
+
+#define _PyUnicode_NONCOMPACT_DATA(op) \
+ (assert(((PyUnicodeObject*)(op))->data.any), \
+ ((((PyUnicodeObject *)(op))->data.any)))
+
+#define PyUnicode_DATA(op) \
+ (assert(PyUnicode_Check(op)), \
+ PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \
+ _PyUnicode_NONCOMPACT_DATA(op))
+
+/* In the access macros below, "kind" may be evaluated more than once.
+ All other macro parameters are evaluated exactly once, so it is safe
+ to put side effects into them (such as increasing the index). */
+
+/* Write into the canonical representation, this macro does not do any sanity
+ checks and is intended for usage in loops. The caller should cache the
+ kind and data pointers obtained from other macro calls.
+ index is the index in the string (starts at 0) and value is the new
+ code point value which should be written to that location. */
+#define PyUnicode_WRITE(kind, data, index, value) \
+ do { \
+ switch ((kind)) { \
+ case PyUnicode_1BYTE_KIND: { \
+ ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \
+ break; \
+ } \
+ case PyUnicode_2BYTE_KIND: { \
+ ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \
+ break; \
+ } \
+ default: { \
+ assert((kind) == PyUnicode_4BYTE_KIND); \
+ ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \
+ } \
+ } \
+ } while (0)
+
+/* Read a code point from the string's canonical representation. No checks
+ or ready calls are performed. */
+#define PyUnicode_READ(kind, data, index) \
+ ((Py_UCS4) \
+ ((kind) == PyUnicode_1BYTE_KIND ? \
+ ((const Py_UCS1 *)(data))[(index)] : \
+ ((kind) == PyUnicode_2BYTE_KIND ? \
+ ((const Py_UCS2 *)(data))[(index)] : \
+ ((const Py_UCS4 *)(data))[(index)] \
+ ) \
+ ))
+
+/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it
+ calls PyUnicode_KIND() and might call it twice. For single reads, use
+ PyUnicode_READ_CHAR, for multiple consecutive reads callers should
+ cache kind and use PyUnicode_READ instead. */
+#define PyUnicode_READ_CHAR(unicode, index) \
+ (assert(PyUnicode_Check(unicode)), \
+ assert(PyUnicode_IS_READY(unicode)), \
+ (Py_UCS4) \
+ (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \
+ ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \
+ (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \
+ ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \
+ ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \
+ ) \
+ ))
+
+/* Returns the length of the unicode string. The caller has to make sure that
+ the string has it's canonical representation set before calling
+ this macro. Call PyUnicode_(FAST_)Ready to ensure that. */
+#define PyUnicode_GET_LENGTH(op) \
+ (assert(PyUnicode_Check(op)), \
+ assert(PyUnicode_IS_READY(op)), \
+ ((PyASCIIObject *)(op))->length)
+
+
+/* Fast check to determine whether an object is ready. Equivalent to
+ PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */
+
+#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready)
+
+/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best
+ case. If the canonical representation is not yet set, it will still call
+ _PyUnicode_Ready().
+ Returns 0 on success and -1 on errors. */
+#define PyUnicode_READY(op) \
+ (assert(PyUnicode_Check(op)), \
+ (PyUnicode_IS_READY(op) ? \
+ 0 : _PyUnicode_Ready(_PyObject_CAST(op))))
+
+/* Return a maximum character value which is suitable for creating another
+ string based on op. This is always an approximation but more efficient
+ than iterating over the string. */
+#define PyUnicode_MAX_CHAR_VALUE(op) \
+ (assert(PyUnicode_IS_READY(op)), \
+ (PyUnicode_IS_ASCII(op) ? \
+ (0x7f) : \
+ (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \
+ (0xffU) : \
+ (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \
+ (0xffffU) : \
+ (0x10ffffU)))))
+
+/* === Public API ========================================================= */
+
+/* --- Plain Py_UNICODE --------------------------------------------------- */
+
+/* With PEP 393, this is the recommended way to allocate a new unicode object.
+ This function will allocate the object and its buffer in a single memory
+ block. Objects created using this function are not resizable. */
+PyAPI_FUNC(PyObject*) PyUnicode_New(
+ Py_ssize_t size, /* Number of code points in the new string */
+ Py_UCS4 maxchar /* maximum code point value in the string */
+ );
+
+/* Initializes the canonical string representation from the deprecated
+ wstr/Py_UNICODE representation. This function is used to convert Unicode
+ objects which were created using the old API to the new flexible format
+ introduced with PEP 393.
+
+ Don't call this function directly, use the public PyUnicode_READY() macro
+ instead. */
+PyAPI_FUNC(int) _PyUnicode_Ready(
+ PyObject *unicode /* Unicode object */
+ );
+
+/* Get a copy of a Unicode string. */
+PyAPI_FUNC(PyObject*) _PyUnicode_Copy(
+ PyObject *unicode
+ );
+
+/* Copy character from one unicode object into another, this function performs
+ character conversion when necessary and falls back to memcpy() if possible.
+
+ Fail if to is too small (smaller than *how_many* or smaller than
+ len(from)-from_start), or if kind(from[from_start:from_start+how_many]) >
+ kind(to), or if *to* has more than 1 reference.
+
+ Return the number of written character, or return -1 and raise an exception
+ on error.
+
+ Pseudo-code:
+
+ how_many = min(how_many, len(from) - from_start)
+ to[to_start:to_start+how_many] = from[from_start:from_start+how_many]
+ return how_many
+
+ Note: The function doesn't write a terminating null character.
+ */
+PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters(
+ PyObject *to,
+ Py_ssize_t to_start,
+ PyObject *from,
+ Py_ssize_t from_start,
+ Py_ssize_t how_many
+ );
+
+/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so
+ may crash if parameters are invalid (e.g. if the output string
+ is too short). */
+PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters(
+ PyObject *to,
+ Py_ssize_t to_start,
+ PyObject *from,
+ Py_ssize_t from_start,
+ Py_ssize_t how_many
+ );
+
+/* Fill a string with a character: write fill_char into
+ unicode[start:start+length].
+
+ Fail if fill_char is bigger than the string maximum character, or if the
+ string has more than 1 reference.
+
+ Return the number of written character, or return -1 and raise an exception
+ on error. */
+PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill(
+ PyObject *unicode,
+ Py_ssize_t start,
+ Py_ssize_t length,
+ Py_UCS4 fill_char
+ );
+
+/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash
+ if parameters are invalid (e.g. if length is longer than the string). */
+PyAPI_FUNC(void) _PyUnicode_FastFill(
+ PyObject *unicode,
+ Py_ssize_t start,
+ Py_ssize_t length,
+ Py_UCS4 fill_char
+ );
+
+/* Create a Unicode Object from the Py_UNICODE buffer u of the given
+ size.
+
+ u may be NULL which causes the contents to be undefined. It is the
+ user's responsibility to fill in the needed data afterwards. Note
+ that modifying the Unicode object contents after construction is
+ only allowed if u was set to NULL.
+
+ The buffer is copied into the new object. */
+PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
+ const Py_UNICODE *u, /* Unicode buffer */
+ Py_ssize_t size /* size of buffer */
+ ) /* Py_DEPRECATED(3.3) */;
+
+/* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters.
+ Scan the string to find the maximum character. */
+PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData(
+ int kind,
+ const void *buffer,
+ Py_ssize_t size);
+
+/* Create a new string from a buffer of ASCII characters.
+ WARNING: Don't check if the string contains any non-ASCII character. */
+PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII(
+ const char *buffer,
+ Py_ssize_t size);
+
+/* Compute the maximum character of the substring unicode[start:end].
+ Return 127 for an empty string. */
+PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar (
+ PyObject *unicode,
+ Py_ssize_t start,
+ Py_ssize_t end);
+
+/* Return a read-only pointer to the Unicode object's internal
+ Py_UNICODE buffer.
+ If the wchar_t/Py_UNICODE representation is not yet available, this
+ function will calculate it. */
+PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
+ PyObject *unicode /* Unicode object */
+ ) /* Py_DEPRECATED(3.3) */;
+
+/* Similar to PyUnicode_AsUnicode(), but raises a ValueError if the string
+ contains null characters. */
+PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode(
+ PyObject *unicode /* Unicode object */
+ );
+
+/* Return a read-only pointer to the Unicode object's internal
+ Py_UNICODE buffer and save the length at size.
+ If the wchar_t/Py_UNICODE representation is not yet available, this
+ function will calculate it. */
+
+PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize(
+ PyObject *unicode, /* Unicode object */
+ Py_ssize_t *size /* location where to save the length */
+ ) /* Py_DEPRECATED(3.3) */;
+
+/* Get the maximum ordinal for a Unicode character. */
+PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void) Py_DEPRECATED(3.3);
+
+
+/* --- _PyUnicodeWriter API ----------------------------------------------- */
+
+typedef struct {
+ PyObject *buffer;
+ void *data;
+ enum PyUnicode_Kind kind;
+ Py_UCS4 maxchar;
+ Py_ssize_t size;
+ Py_ssize_t pos;
+
+ /* minimum number of allocated characters (default: 0) */
+ Py_ssize_t min_length;
+
+ /* minimum character (default: 127, ASCII) */
+ Py_UCS4 min_char;
+
+ /* If non-zero, overallocate the buffer (default: 0). */
+ unsigned char overallocate;
+
+ /* If readonly is 1, buffer is a shared string (cannot be modified)
+ and size is set to 0. */
+ unsigned char readonly;
+} _PyUnicodeWriter ;
+
+/* Initialize a Unicode writer.
+ *
+ * By default, the minimum buffer size is 0 character and overallocation is
+ * disabled. Set min_length, min_char and overallocate attributes to control
+ * the allocation of the buffer. */
+PyAPI_FUNC(void)
+_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
+
+/* Prepare the buffer to write 'length' characters
+ with the specified maximum character.
+
+ Return 0 on success, raise an exception and return -1 on error. */
+#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \
+ (((MAXCHAR) <= (WRITER)->maxchar \
+ && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \
+ ? 0 \
+ : (((LENGTH) == 0) \
+ ? 0 \
+ : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR))))
+
+/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro
+ instead. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
+ Py_ssize_t length, Py_UCS4 maxchar);
+
+/* Prepare the buffer to have at least the kind KIND.
+ For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will
+ support characters in range U+000-U+FFFF.
+
+ Return 0 on success, raise an exception and return -1 on error. */
+#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \
+ (assert((KIND) != PyUnicode_WCHAR_KIND), \
+ (KIND) <= (WRITER)->kind \
+ ? 0 \
+ : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND)))
+
+/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind()
+ macro instead. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer,
+ enum PyUnicode_Kind kind);
+
+/* Append a Unicode character.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
+ Py_UCS4 ch
+ );
+
+/* Append a Unicode string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
+ PyObject *str /* Unicode string */
+ );
+
+/* Append a substring of a Unicode string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
+ PyObject *str, /* Unicode string */
+ Py_ssize_t start,
+ Py_ssize_t end
+ );
+
+/* Append an ASCII-encoded byte string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
+ const char *str, /* ASCII-encoded byte string */
+ Py_ssize_t len /* number of bytes, or -1 if unknown */
+ );
+
+/* Append a latin1-encoded byte string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
+ const char *str, /* latin1-encoded byte string */
+ Py_ssize_t len /* length in bytes */
+ );
+
+/* Get the value of the writer as a Unicode string. Clear the
+ buffer of the writer. Raise an exception and return NULL
+ on error. */
+PyAPI_FUNC(PyObject *)
+_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
+
+/* Deallocate memory of a writer (clear its internal buffer). */
+PyAPI_FUNC(void)
+_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
+
+
+/* Format the object based on the format_spec, as defined in PEP 3101
+ (Advanced String Formatting). */
+PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter(
+ _PyUnicodeWriter *writer,
+ PyObject *obj,
+ PyObject *format_spec,
+ Py_ssize_t start,
+ Py_ssize_t end);
+
+PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void);
+
+/* --- wchar_t support for platforms which support it --------------------- */
+
+#ifdef HAVE_WCHAR_H
+PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind);
+#endif
+
+/* --- Manage the default encoding ---------------------------------------- */
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+ Unicode object unicode and the size of the encoded representation
+ in bytes stored in *size.
+
+ In case of an error, no *size is set.
+
+ This function caches the UTF-8 encoded string in the unicodeobject
+ and subsequent calls will return the same string. The memory is released
+ when the unicodeobject is deallocated.
+
+ _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to
+ support the previous internal function with the same behaviour.
+
+ *** This API is for interpreter INTERNAL USE ONLY and will likely
+ *** be removed or changed in the future.
+
+ *** If you need to access the Unicode object as UTF-8 bytes string,
+ *** please use PyUnicode_AsUTF8String() instead.
+*/
+
+PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize(
+ PyObject *unicode,
+ Py_ssize_t *size);
+
+#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize
+
+/* Returns a pointer to the default encoding (UTF-8) of the
+ Unicode object unicode.
+
+ Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation
+ in the unicodeobject.
+
+ _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to
+ support the previous internal function with the same behaviour.
+
+ Use of this API is DEPRECATED since no size information can be
+ extracted from the returned data.
+
+ *** This API is for interpreter INTERNAL USE ONLY and will likely
+ *** be removed or changed for Python 3.1.
+
+ *** If you need to access the Unicode object as UTF-8 bytes string,
+ *** please use PyUnicode_AsUTF8String() instead.
+
+*/
+
+PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode);
+
+#define _PyUnicode_AsString PyUnicode_AsUTF8
+
+/* --- Generic Codecs ----------------------------------------------------- */
+
+/* Encodes a Py_UNICODE buffer of the given size and returns a
+ Python string object. */
+PyAPI_FUNC(PyObject*) PyUnicode_Encode(
+ const Py_UNICODE *s, /* Unicode char buffer */
+ Py_ssize_t size, /* number of Py_UNICODE chars to encode */
+ const char *encoding, /* encoding */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+/* --- UTF-7 Codecs ------------------------------------------------------- */
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
+ int base64SetO, /* Encode RFC2152 Set O characters in base64 */
+ int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7(
+ PyObject *unicode, /* Unicode object */
+ int base64SetO, /* Encode RFC2152 Set O characters in base64 */
+ int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
+ const char *errors /* error handling */
+ );
+
+/* --- UTF-8 Codecs ------------------------------------------------------- */
+
+PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
+ PyObject *unicode,
+ const char *errors);
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+/* --- UTF-32 Codecs ------------------------------------------------------ */
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
+ const char *errors, /* error handling */
+ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
+ PyObject *object, /* Unicode object */
+ const char *errors, /* error handling */
+ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
+ );
+
+/* --- UTF-16 Codecs ------------------------------------------------------ */
+
+/* Returns a Python string object holding the UTF-16 encoded value of
+ the Unicode data.
+
+ If byteorder is not 0, output is written according to the following
+ byte order:
+
+ byteorder == -1: little endian
+ byteorder == 0: native byte order (writes a BOM mark)
+ byteorder == 1: big endian
+
+ If byteorder is 0, the output string will always start with the
+ Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
+ prepended.
+
+ Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
+ UCS-2. This trick makes it possible to add full UTF-16 capabilities
+ at a later point without compromising the APIs.
+
+*/
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
+ const char *errors, /* error handling */
+ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(
+ PyObject* unicode, /* Unicode object */
+ const char *errors, /* error handling */
+ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
+ );
+
+/* --- Unicode-Escape Codecs ---------------------------------------------- */
+
+/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
+ chars. */
+PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape(
+ const char *string, /* Unicode-Escape encoded string */
+ Py_ssize_t length, /* size of string */
+ const char *errors, /* error handling */
+ const char **first_invalid_escape /* on return, points to first
+ invalid escaped char in
+ string. */
+);
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length /* Number of Py_UNICODE chars to encode */
+ ) Py_DEPRECATED(3.3);
+
+/* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length /* Number of Py_UNICODE chars to encode */
+ ) Py_DEPRECATED(3.3);
+
+/* --- Unicode Internal Codec --------------------------------------------- */
+
+/* Only for internal use in _codecsmodule.c */
+PyObject *_PyUnicode_DecodeUnicodeInternal(
+ const char *string,
+ Py_ssize_t length,
+ const char *errors
+ );
+
+/* --- Latin-1 Codecs ----------------------------------------------------- */
+
+PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String(
+ PyObject* unicode,
+ const char* errors);
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+/* --- ASCII Codecs ------------------------------------------------------- */
+
+PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString(
+ PyObject* unicode,
+ const char* errors);
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+/* --- Character Map Codecs ----------------------------------------------- */
+
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ PyObject *mapping, /* encoding mapping */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap(
+ PyObject *unicode, /* Unicode object */
+ PyObject *mapping, /* encoding mapping */
+ const char *errors /* error handling */
+ );
+
+/* Translate a Py_UNICODE buffer of the given length by applying a
+ character mapping table to it and return the resulting Unicode
+ object.
+
+ The mapping table must map Unicode ordinal integers to Unicode strings,
+ Unicode ordinal integers or None (causing deletion of the character).
+
+ Mapping tables may be dictionaries or sequences. Unmapped character
+ ordinals (ones which cause a LookupError) are left untouched and
+ are copied as-is.
+
+*/
+PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ PyObject *table, /* Translate table */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+
+/* --- MBCS codecs for Windows -------------------------------------------- */
+
+#ifdef MS_WINDOWS
+PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
+ const Py_UNICODE *data, /* Unicode char buffer */
+ Py_ssize_t length, /* number of Py_UNICODE chars to encode */
+ const char *errors /* error handling */
+ ) Py_DEPRECATED(3.3);
+#endif
+
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+/* Takes a Unicode string holding a decimal value and writes it into
+ an output buffer using standard ASCII digit codes.
+
+ The output buffer has to provide at least length+1 bytes of storage
+ area. The output string is 0-terminated.
+
+ The encoder converts whitespace to ' ', decimal characters to their
+ corresponding ASCII digit and all other Latin-1 characters except
+ \0 as-is. Characters outside this range (Unicode ordinals 1-256)
+ are treated as errors. This includes embedded NULL bytes.
+
+ Error handling is defined by the errors argument:
+
+ NULL or "strict": raise a ValueError
+ "ignore": ignore the wrong characters (these are not copied to the
+ output buffer)
+ "replace": replaces illegal characters with '?'
+
+ Returns 0 on success, -1 on failure.
+
+*/
+
+PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
+ Py_UNICODE *s, /* Unicode buffer */
+ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
+ char *output, /* Output buffer; must have size >= length */
+ const char *errors /* error handling */
+ ) /* Py_DEPRECATED(3.3) */;
+
+/* Transforms code points that have decimal digit property to the
+ corresponding ASCII digit code points.
+
+ Returns a new Unicode string on success, NULL on failure.
+*/
+
+PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII(
+ Py_UNICODE *s, /* Unicode buffer */
+ Py_ssize_t length /* Number of Py_UNICODE chars to transform */
+ ) /* Py_DEPRECATED(3.3) */;
+
+/* Coverts a Unicode object holding a decimal value to an ASCII string
+ for using in int, float and complex parsers.
+ Transforms code points that have decimal digit property to the
+ corresponding ASCII digit code points. Transforms spaces to ASCII.
+ Transforms code points starting from the first non-ASCII code point that
+ is neither a decimal digit nor a space to the end into '?'. */
+
+PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
+ PyObject *unicode /* Unicode object */
+ );
+
+/* --- Methods & Slots ---------------------------------------------------- */
+
+PyAPI_FUNC(PyObject *) _PyUnicode_JoinArray(
+ PyObject *separator,
+ PyObject *const *items,
+ Py_ssize_t seqlen
+ );
+
+/* Test whether a unicode is equal to ASCII identifier. Return 1 if true,
+ 0 otherwise. The right argument must be ASCII identifier.
+ Any error occurs inside will be cleared before return. */
+PyAPI_FUNC(int) _PyUnicode_EqualToASCIIId(
+ PyObject *left, /* Left string */
+ _Py_Identifier *right /* Right identifier */
+ );
+
+/* Test whether a unicode is equal to ASCII string. Return 1 if true,
+ 0 otherwise. The right argument must be ASCII-encoded string.
+ Any error occurs inside will be cleared before return. */
+PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(
+ PyObject *left,
+ const char *right /* ASCII-encoded string */
+ );
+
+/* Externally visible for str.strip(unicode) */
+PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
+ PyObject *self,
+ int striptype,
+ PyObject *sepobj
+ );
+
+/* Using explicit passed-in values, insert the thousands grouping
+ into the string pointed to by buffer. For the argument descriptions,
+ see Objects/stringlib/localeutil.h */
+PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
+ _PyUnicodeWriter *writer,
+ Py_ssize_t n_buffer,
+ PyObject *digits,
+ Py_ssize_t d_pos,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width,
+ const char *grouping,
+ PyObject *thousands_sep,
+ Py_UCS4 *maxchar);
+
+/* === Characters Type APIs =============================================== */
+
+/* Helper array used by Py_UNICODE_ISSPACE(). */
+
+PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
+
+/* These should not be used directly. Use the Py_UNICODE_IS* and
+ Py_UNICODE_TO* macros instead.
+
+ These APIs are implemented in Objects/unicodectype.c.
+
+*/
+
+PyAPI_FUNC(int) _PyUnicode_IsLowercase(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsUppercase(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsXidStart(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsXidContinue(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
+ const Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
+ const Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase(
+ Py_UCS4 ch /* Unicode character */
+ ) /* Py_DEPRECATED(3.3) */;
+
+PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase(
+ Py_UCS4 ch /* Unicode character */
+ ) /* Py_DEPRECATED(3.3) */;
+
+PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase(
+ Py_UCS4 ch /* Unicode character */
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(int) _PyUnicode_ToLowerFull(
+ Py_UCS4 ch, /* Unicode character */
+ Py_UCS4 *res
+ );
+
+PyAPI_FUNC(int) _PyUnicode_ToTitleFull(
+ Py_UCS4 ch, /* Unicode character */
+ Py_UCS4 *res
+ );
+
+PyAPI_FUNC(int) _PyUnicode_ToUpperFull(
+ Py_UCS4 ch, /* Unicode character */
+ Py_UCS4 *res
+ );
+
+PyAPI_FUNC(int) _PyUnicode_ToFoldedFull(
+ Py_UCS4 ch, /* Unicode character */
+ Py_UCS4 *res
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsCased(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_ToDigit(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(double) _PyUnicode_ToNumeric(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsDigit(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsNumeric(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsPrintable(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(int) _PyUnicode_IsAlpha(
+ Py_UCS4 ch /* Unicode character */
+ );
+
+PyAPI_FUNC(size_t) Py_UNICODE_strlen(
+ const Py_UNICODE *u
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcpy(
+ Py_UNICODE *s1,
+ const Py_UNICODE *s2) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcat(
+ Py_UNICODE *s1, const Py_UNICODE *s2) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strncpy(
+ Py_UNICODE *s1,
+ const Py_UNICODE *s2,
+ size_t n) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(int) Py_UNICODE_strcmp(
+ const Py_UNICODE *s1,
+ const Py_UNICODE *s2
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(int) Py_UNICODE_strncmp(
+ const Py_UNICODE *s1,
+ const Py_UNICODE *s2,
+ size_t n
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strchr(
+ const Py_UNICODE *s,
+ Py_UNICODE c
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strrchr(
+ const Py_UNICODE *s,
+ Py_UNICODE c
+ ) Py_DEPRECATED(3.3);
+
+PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int);
+
+/* Create a copy of a unicode string ending with a nul character. Return NULL
+ and raise a MemoryError exception on memory allocation failure, otherwise
+ return a new allocated buffer (use PyMem_Free() to free the buffer). */
+
+PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy(
+ PyObject *unicode
+ ) Py_DEPRECATED(3.3);
+
+/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/
+PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
+/* Clear all static strings. */
+PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void);
+
+/* Fast equality check when the inputs are known to be exact unicode types
+ and where the hash values are equal (i.e. a very probable match) */
+PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index e7192852f045..503aeb5d19a6 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -83,16 +83,6 @@ Copyright (c) Corporation for National Research Initiatives.
/* #define HAVE_WCHAR_H */
/* #define HAVE_USABLE_WCHAR_T */
-/* Py_UNICODE was the native Unicode storage format (code unit) used by
- Python and represents a single Unicode element in the Unicode type.
- With PEP 393, Py_UNICODE is deprecated and replaced with a
- typedef to wchar_t. */
-
-#ifndef Py_LIMITED_API
-#define PY_UNICODE_TYPE wchar_t
-typedef wchar_t Py_UNICODE /* Py_DEPRECATED(3.3) */;
-#endif
-
/* If the compiler provides a wchar_t type we try to support it
through the interface functions PyUnicode_FromWideChar(),
PyUnicode_AsWideChar() and PyUnicode_AsWideCharString(). */
@@ -113,248 +103,10 @@ typedef uint32_t Py_UCS4;
typedef uint16_t Py_UCS2;
typedef uint8_t Py_UCS1;
-/* --- Internal Unicode Operations ---------------------------------------- */
-
-/* Since splitting on whitespace is an important use case, and
- whitespace in most situations is solely ASCII whitespace, we
- optimize for the common case by using a quick look-up table
- _Py_ascii_whitespace (see below) with an inlined check.
-
- */
-#ifndef Py_LIMITED_API
-#define Py_UNICODE_ISSPACE(ch) \
- ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
-
-#define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
-#define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
-#define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
-#define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
-
-#define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
-#define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
-#define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
-
-#define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
-#define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
-#define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
-#define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch)
-
-#define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
-#define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
-#define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
-
-#define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
-
-#define Py_UNICODE_ISALNUM(ch) \
- (Py_UNICODE_ISALPHA(ch) || \
- Py_UNICODE_ISDECIMAL(ch) || \
- Py_UNICODE_ISDIGIT(ch) || \
- Py_UNICODE_ISNUMERIC(ch))
-
-#define Py_UNICODE_COPY(target, source, length) \
- memcpy((target), (source), (length)*sizeof(Py_UNICODE))
-
-#define Py_UNICODE_FILL(target, value, length) \
- do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
- for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
- } while (0)
-
-/* macros to work with surrogates */
-#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF)
-#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF)
-#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF)
-/* Join two surrogate characters and return a single Py_UCS4 value. */
-#define Py_UNICODE_JOIN_SURROGATES(high, low) \
- (((((Py_UCS4)(high) & 0x03FF) << 10) | \
- ((Py_UCS4)(low) & 0x03FF)) + 0x10000)
-/* high surrogate = top 10 bits added to D800 */
-#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10))
-/* low surrogate = bottom 10 bits added to DC00 */
-#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF))
-
-/* Check if substring matches at given offset. The offset must be
- valid, and the substring must not be empty. */
-
-#define Py_UNICODE_MATCH(string, offset, substring) \
- ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \
- ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \
- !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE)))
-
-#endif /* Py_LIMITED_API */
-
#ifdef __cplusplus
extern "C" {
#endif
-/* --- Unicode Type ------------------------------------------------------- */
-
-#ifndef Py_LIMITED_API
-
-/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject
- structure. state.ascii and state.compact are set, and the data
- immediately follow the structure. utf8_length and wstr_length can be found
- in the length field; the utf8 pointer is equal to the data pointer. */
-typedef struct {
- /* There are 4 forms of Unicode strings:
-
- - compact ascii:
-
- * structure = PyASCIIObject
- * test: PyUnicode_IS_COMPACT_ASCII(op)
- * kind = PyUnicode_1BYTE_KIND
- * compact = 1
- * ascii = 1
- * ready = 1
- * (length is the length of the utf8 and wstr strings)
- * (data starts just after the structure)
- * (since ASCII is decoded from UTF-8, the utf8 string are the data)
-
- - compact:
-
- * structure = PyCompactUnicodeObject
- * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op)
- * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
- PyUnicode_4BYTE_KIND
- * compact = 1
- * ready = 1
- * ascii = 0
- * utf8 is not shared with data
- * utf8_length = 0 if utf8 is NULL
- * wstr is shared with data and wstr_length=length
- if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
- or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4
- * wstr_length = 0 if wstr is NULL
- * (data starts just after the structure)
-
- - legacy string, not ready:
-
- * structure = PyUnicodeObject
- * test: kind == PyUnicode_WCHAR_KIND
- * length = 0 (use wstr_length)
- * hash = -1
- * kind = PyUnicode_WCHAR_KIND
- * compact = 0
- * ascii = 0
- * ready = 0
- * interned = SSTATE_NOT_INTERNED
- * wstr is not NULL
- * data.any is NULL
- * utf8 is NULL
- * utf8_length = 0
-
- - legacy string, ready:
-
- * structure = PyUnicodeObject structure
- * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND
- * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or
- PyUnicode_4BYTE_KIND
- * compact = 0
- * ready = 1
- * data.any is not NULL
- * utf8 is shared and utf8_length = length with data.any if ascii = 1
- * utf8_length = 0 if utf8 is NULL
- * wstr is shared with data.any and wstr_length = length
- if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2
- or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4
- * wstr_length = 0 if wstr is NULL
-
- Compact strings use only one memory block (structure + characters),
- whereas legacy strings use one block for the structure and one block
- for characters.
-
- Legacy strings are created by PyUnicode_FromUnicode() and
- PyUnicode_FromStringAndSize(NULL, size) functions. They become ready
- when PyUnicode_READY() is called.
-
- See also _PyUnicode_CheckConsistency().
- */
- PyObject_HEAD
- Py_ssize_t length; /* Number of code points in the string */
- Py_hash_t hash; /* Hash value; -1 if not set */
- struct {
- /*
- SSTATE_NOT_INTERNED (0)
- SSTATE_INTERNED_MORTAL (1)
- SSTATE_INTERNED_IMMORTAL (2)
-
- If interned != SSTATE_NOT_INTERNED, the two references from the
- dictionary to this object are *not* counted in ob_refcnt.
- */
- unsigned int interned:2;
- /* Character size:
-
- - PyUnicode_WCHAR_KIND (0):
-
- * character type = wchar_t (16 or 32 bits, depending on the
- platform)
-
- - PyUnicode_1BYTE_KIND (1):
-
- * character type = Py_UCS1 (8 bits, unsigned)
- * all characters are in the range U+0000-U+00FF (latin1)
- * if ascii is set, all characters are in the range U+0000-U+007F
- (ASCII), otherwise at least one character is in the range
- U+0080-U+00FF
-
- - PyUnicode_2BYTE_KIND (2):
-
- * character type = Py_UCS2 (16 bits, unsigned)
- * all characters are in the range U+0000-U+FFFF (BMP)
- * at least one character is in the range U+0100-U+FFFF
-
- - PyUnicode_4BYTE_KIND (4):
-
- * character type = Py_UCS4 (32 bits, unsigned)
- * all characters are in the range U+0000-U+10FFFF
- * at least one character is in the range U+10000-U+10FFFF
- */
- unsigned int kind:3;
- /* Compact is with respect to the allocation scheme. Compact unicode
- objects only require one memory block while non-compact objects use
- one block for the PyUnicodeObject struct and another for its data
- buffer. */
- unsigned int compact:1;
- /* The string only contains characters in the range U+0000-U+007F (ASCII)
- and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is
- set, use the PyASCIIObject structure. */
- unsigned int ascii:1;
- /* The ready flag indicates whether the object layout is initialized
- completely. This means that this is either a compact object, or
- the data pointer is filled out. The bit is redundant, and helps
- to minimize the test in PyUnicode_IS_READY(). */
- unsigned int ready:1;
- /* Padding to ensure that PyUnicode_DATA() is always aligned to
- 4 bytes (see issue #19537 on m68k). */
- unsigned int :24;
- } state;
- wchar_t *wstr; /* wchar_t representation (null-terminated) */
-} PyASCIIObject;
-
-/* Non-ASCII strings allocated through PyUnicode_New use the
- PyCompactUnicodeObject structure. state.compact is set, and the data
- immediately follow the structure. */
-typedef struct {
- PyASCIIObject _base;
- Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the
- * terminating \0. */
- char *utf8; /* UTF-8 representation (null-terminated) */
- Py_ssize_t wstr_length; /* Number of code points in wstr, possible
- * surrogates count as two code points. */
-} PyCompactUnicodeObject;
-
-/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the
- PyUnicodeObject structure. The actual string data is initially in the wstr
- block, and copied into the data block using _PyUnicode_Ready. */
-typedef struct {
- PyCompactUnicodeObject _base;
- union {
- void *any;
- Py_UCS1 *latin1;
- Py_UCS2 *ucs2;
- Py_UCS4 *ucs4;
- } data; /* Canonical, smallest-form Unicode buffer */
-} PyUnicodeObject;
-#endif
PyAPI_DATA(PyTypeObject) PyUnicode_Type;
PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type;
@@ -363,209 +115,6 @@ PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type;
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
#define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
-/* Fast access macros */
-#ifndef Py_LIMITED_API
-
-#define PyUnicode_WSTR_LENGTH(op) \
- (PyUnicode_IS_COMPACT_ASCII(op) ? \
- ((PyASCIIObject*)op)->length : \
- ((PyCompactUnicodeObject*)op)->wstr_length)
-
-/* Returns the deprecated Py_UNICODE representation's size in code units
- (this includes surrogate pairs as 2 units).
- If the Py_UNICODE representation is not available, it will be computed
- on request. Use PyUnicode_GET_LENGTH() for the length in code points. */
-
-#define PyUnicode_GET_SIZE(op) \
- (assert(PyUnicode_Check(op)), \
- (((PyASCIIObject *)(op))->wstr) ? \
- PyUnicode_WSTR_LENGTH(op) : \
- ((void)PyUnicode_AsUnicode(_PyObject_CAST(op)),\
- assert(((PyASCIIObject *)(op))->wstr), \
- PyUnicode_WSTR_LENGTH(op)))
- /* Py_DEPRECATED(3.3) */
-
-#define PyUnicode_GET_DATA_SIZE(op) \
- (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE)
- /* Py_DEPRECATED(3.3) */
-
-/* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE
- representation on demand. Using this macro is very inefficient now,
- try to port your code to use the new PyUnicode_*BYTE_DATA() macros or
- use PyUnicode_WRITE() and PyUnicode_READ(). */
-
-#define PyUnicode_AS_UNICODE(op) \
- (assert(PyUnicode_Check(op)), \
- (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \
- PyUnicode_AsUnicode(_PyObject_CAST(op)))
- /* Py_DEPRECATED(3.3) */
-
-#define PyUnicode_AS_DATA(op) \
- ((const char *)(PyUnicode_AS_UNICODE(op)))
- /* Py_DEPRECATED(3.3) */
-
-
-/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */
-
-/* Values for PyASCIIObject.state: */
-
-/* Interning state. */
-#define SSTATE_NOT_INTERNED 0
-#define SSTATE_INTERNED_MORTAL 1
-#define SSTATE_INTERNED_IMMORTAL 2
-
-/* Return true if the string contains only ASCII characters, or 0 if not. The
- string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be
- ready. */
-#define PyUnicode_IS_ASCII(op) \
- (assert(PyUnicode_Check(op)), \
- assert(PyUnicode_IS_READY(op)), \
- ((PyASCIIObject*)op)->state.ascii)
-
-/* Return true if the string is compact or 0 if not.
- No type checks or Ready calls are performed. */
-#define PyUnicode_IS_COMPACT(op) \
- (((PyASCIIObject*)(op))->state.compact)
-
-/* Return true if the string is a compact ASCII string (use PyASCIIObject
- structure), or 0 if not. No type checks or Ready calls are performed. */
-#define PyUnicode_IS_COMPACT_ASCII(op) \
- (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op))
-
-enum PyUnicode_Kind {
-/* String contains only wstr byte characters. This is only possible
- when the string was created with a legacy API and _PyUnicode_Ready()
- has not been called yet. */
- PyUnicode_WCHAR_KIND = 0,
-/* Return values of the PyUnicode_KIND() macro: */
- PyUnicode_1BYTE_KIND = 1,
- PyUnicode_2BYTE_KIND = 2,
- PyUnicode_4BYTE_KIND = 4
-};
-
-/* Return pointers to the canonical representation cast to unsigned char,
- Py_UCS2, or Py_UCS4 for direct character access.
- No checks are performed, use PyUnicode_KIND() before to ensure
- these will work correctly. */
-
-#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op))
-#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op))
-#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op))
-
-/* Return one of the PyUnicode_*_KIND values defined above. */
-#define PyUnicode_KIND(op) \
- (assert(PyUnicode_Check(op)), \
- assert(PyUnicode_IS_READY(op)), \
- ((PyASCIIObject *)(op))->state.kind)
-
-/* Return a void pointer to the raw unicode buffer. */
-#define _PyUnicode_COMPACT_DATA(op) \
- (PyUnicode_IS_ASCII(op) ? \
- ((void*)((PyASCIIObject*)(op) + 1)) : \
- ((void*)((PyCompactUnicodeObject*)(op) + 1)))
-
-#define _PyUnicode_NONCOMPACT_DATA(op) \
- (assert(((PyUnicodeObject*)(op))->data.any), \
- ((((PyUnicodeObject *)(op))->data.any)))
-
-#define PyUnicode_DATA(op) \
- (assert(PyUnicode_Check(op)), \
- PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \
- _PyUnicode_NONCOMPACT_DATA(op))
-
-/* In the access macros below, "kind" may be evaluated more than once.
- All other macro parameters are evaluated exactly once, so it is safe
- to put side effects into them (such as increasing the index). */
-
-/* Write into the canonical representation, this macro does not do any sanity
- checks and is intended for usage in loops. The caller should cache the
- kind and data pointers obtained from other macro calls.
- index is the index in the string (starts at 0) and value is the new
- code point value which should be written to that location. */
-#define PyUnicode_WRITE(kind, data, index, value) \
- do { \
- switch ((kind)) { \
- case PyUnicode_1BYTE_KIND: { \
- ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \
- break; \
- } \
- case PyUnicode_2BYTE_KIND: { \
- ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \
- break; \
- } \
- default: { \
- assert((kind) == PyUnicode_4BYTE_KIND); \
- ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \
- } \
- } \
- } while (0)
-
-/* Read a code point from the string's canonical representation. No checks
- or ready calls are performed. */
-#define PyUnicode_READ(kind, data, index) \
- ((Py_UCS4) \
- ((kind) == PyUnicode_1BYTE_KIND ? \
- ((const Py_UCS1 *)(data))[(index)] : \
- ((kind) == PyUnicode_2BYTE_KIND ? \
- ((const Py_UCS2 *)(data))[(index)] : \
- ((const Py_UCS4 *)(data))[(index)] \
- ) \
- ))
-
-/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it
- calls PyUnicode_KIND() and might call it twice. For single reads, use
- PyUnicode_READ_CHAR, for multiple consecutive reads callers should
- cache kind and use PyUnicode_READ instead. */
-#define PyUnicode_READ_CHAR(unicode, index) \
- (assert(PyUnicode_Check(unicode)), \
- assert(PyUnicode_IS_READY(unicode)), \
- (Py_UCS4) \
- (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \
- ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \
- (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \
- ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \
- ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \
- ) \
- ))
-
-/* Returns the length of the unicode string. The caller has to make sure that
- the string has it's canonical representation set before calling
- this macro. Call PyUnicode_(FAST_)Ready to ensure that. */
-#define PyUnicode_GET_LENGTH(op) \
- (assert(PyUnicode_Check(op)), \
- assert(PyUnicode_IS_READY(op)), \
- ((PyASCIIObject *)(op))->length)
-
-
-/* Fast check to determine whether an object is ready. Equivalent to
- PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */
-
-#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready)
-
-/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best
- case. If the canonical representation is not yet set, it will still call
- _PyUnicode_Ready().
- Returns 0 on success and -1 on errors. */
-#define PyUnicode_READY(op) \
- (assert(PyUnicode_Check(op)), \
- (PyUnicode_IS_READY(op) ? \
- 0 : _PyUnicode_Ready(_PyObject_CAST(op))))
-
-/* Return a maximum character value which is suitable for creating another
- string based on op. This is always an approximation but more efficient
- than iterating over the string. */
-#define PyUnicode_MAX_CHAR_VALUE(op) \
- (assert(PyUnicode_IS_READY(op)), \
- (PyUnicode_IS_ASCII(op) ? \
- (0x7f) : \
- (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \
- (0xffU) : \
- (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \
- (0xffffU) : \
- (0x10ffffU)))))
-
-#endif
-
/* --- Constants ---------------------------------------------------------- */
/* This Unicode character will be used as replacement character during
@@ -577,120 +126,6 @@ enum PyUnicode_Kind {
/* === Public API ========================================================= */
-/* --- Plain Py_UNICODE --------------------------------------------------- */
-
-/* With PEP 393, this is the recommended way to allocate a new unicode object.
- This function will allocate the object and its buffer in a single memory
- block. Objects created using this function are not resizable. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_New(
- Py_ssize_t size, /* Number of code points in the new string */
- Py_UCS4 maxchar /* maximum code point value in the string */
- );
-#endif
-
-/* Initializes the canonical string representation from the deprecated
- wstr/Py_UNICODE representation. This function is used to convert Unicode
- objects which were created using the old API to the new flexible format
- introduced with PEP 393.
-
- Don't call this function directly, use the public PyUnicode_READY() macro
- instead. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyUnicode_Ready(
- PyObject *unicode /* Unicode object */
- );
-#endif
-
-/* Get a copy of a Unicode string. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) _PyUnicode_Copy(
- PyObject *unicode
- );
-#endif
-
-/* Copy character from one unicode object into another, this function performs
- character conversion when necessary and falls back to memcpy() if possible.
-
- Fail if to is too small (smaller than *how_many* or smaller than
- len(from)-from_start), or if kind(from[from_start:from_start+how_many]) >
- kind(to), or if *to* has more than 1 reference.
-
- Return the number of written character, or return -1 and raise an exception
- on error.
-
- Pseudo-code:
-
- how_many = min(how_many, len(from) - from_start)
- to[to_start:to_start+how_many] = from[from_start:from_start+how_many]
- return how_many
-
- Note: The function doesn't write a terminating null character.
- */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters(
- PyObject *to,
- Py_ssize_t to_start,
- PyObject *from,
- Py_ssize_t from_start,
- Py_ssize_t how_many
- );
-
-/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so
- may crash if parameters are invalid (e.g. if the output string
- is too short). */
-PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters(
- PyObject *to,
- Py_ssize_t to_start,
- PyObject *from,
- Py_ssize_t from_start,
- Py_ssize_t how_many
- );
-#endif
-
-#ifndef Py_LIMITED_API
-/* Fill a string with a character: write fill_char into
- unicode[start:start+length].
-
- Fail if fill_char is bigger than the string maximum character, or if the
- string has more than 1 reference.
-
- Return the number of written character, or return -1 and raise an exception
- on error. */
-PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill(
- PyObject *unicode,
- Py_ssize_t start,
- Py_ssize_t length,
- Py_UCS4 fill_char
- );
-
-/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash
- if parameters are invalid (e.g. if length is longer than the string). */
-PyAPI_FUNC(void) _PyUnicode_FastFill(
- PyObject *unicode,
- Py_ssize_t start,
- Py_ssize_t length,
- Py_UCS4 fill_char
- );
-#endif
-
-/* Create a Unicode Object from the Py_UNICODE buffer u of the given
- size.
-
- u may be NULL which causes the contents to be undefined. It is the
- user's responsibility to fill in the needed data afterwards. Note
- that modifying the Unicode object contents after construction is
- only allowed if u was set to NULL.
-
- The buffer is copied into the new object. */
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
- const Py_UNICODE *u, /* Unicode buffer */
- Py_ssize_t size /* size of buffer */
- ) /* Py_DEPRECATED(3.3) */;
-#endif
-
/* Similar to PyUnicode_FromUnicode(), but u points to UTF-8 encoded bytes */
PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
const char *u, /* UTF-8 encoded string */
@@ -703,21 +138,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromString(
const char *u /* UTF-8 encoded string */
);
-#ifndef Py_LIMITED_API
-/* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters.
- Scan the string to find the maximum character. */
-PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData(
- int kind,
- const void *buffer,
- Py_ssize_t size);
-
-/* Create a new string from a buffer of ASCII characters.
- WARNING: Don't check if the string contains any non-ASCII character. */
-PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII(
- const char *buffer,
- Py_ssize_t size);
-#endif
-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
PyAPI_FUNC(PyObject*) PyUnicode_Substring(
PyObject *str,
@@ -725,15 +145,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_Substring(
Py_ssize_t end);
#endif
-#ifndef Py_LIMITED_API
-/* Compute the maximum character of the substring unicode[start:end].
- Return 127 for an empty string. */
-PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar (
- PyObject *unicode,
- Py_ssize_t start,
- Py_ssize_t end);
-#endif
-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
/* Copy the string into a UCS4 buffer including the null character if copy_null
is set. Return NULL and raise an exception on error. Raise a SystemError if
@@ -752,33 +163,6 @@ PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4(
PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode);
#endif
-#ifndef Py_LIMITED_API
-/* Return a read-only pointer to the Unicode object's internal
- Py_UNICODE buffer.
- If the wchar_t/Py_UNICODE representation is not yet available, this
- function will calculate it. */
-
-PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
- PyObject *unicode /* Unicode object */
- ) /* Py_DEPRECATED(3.3) */;
-
-/* Similar to PyUnicode_AsUnicode(), but raises a ValueError if the string
- contains null characters. */
-PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode(
- PyObject *unicode /* Unicode object */
- );
-
-/* Return a read-only pointer to the Unicode object's internal
- Py_UNICODE buffer and save the length at size.
- If the wchar_t/Py_UNICODE representation is not yet available, this
- function will calculate it. */
-
-PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize(
- PyObject *unicode, /* Unicode object */
- Py_ssize_t *size /* location where to save the length */
- ) /* Py_DEPRECATED(3.3) */;
-#endif
-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
/* Get the length of the Unicode object. */
@@ -814,11 +198,6 @@ PyAPI_FUNC(int) PyUnicode_WriteChar(
);
#endif
-#ifndef Py_LIMITED_API
-/* Get the maximum ordinal for a Unicode character. */
-PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void) Py_DEPRECATED(3.3);
-#endif
-
/* Resize a Unicode object. The length is the number of characters, except
if the kind of the string is PyUnicode_WCHAR_KIND: in this case, the length
is the number of Py_UNICODE characters.
@@ -881,141 +260,11 @@ PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(
...
);
-#ifndef Py_LIMITED_API
-typedef struct {
- PyObject *buffer;
- void *data;
- enum PyUnicode_Kind kind;
- Py_UCS4 maxchar;
- Py_ssize_t size;
- Py_ssize_t pos;
-
- /* minimum number of allocated characters (default: 0) */
- Py_ssize_t min_length;
-
- /* minimum character (default: 127, ASCII) */
- Py_UCS4 min_char;
-
- /* If non-zero, overallocate the buffer (default: 0). */
- unsigned char overallocate;
-
- /* If readonly is 1, buffer is a shared string (cannot be modified)
- and size is set to 0. */
- unsigned char readonly;
-} _PyUnicodeWriter ;
-
-/* Initialize a Unicode writer.
- *
- * By default, the minimum buffer size is 0 character and overallocation is
- * disabled. Set min_length, min_char and overallocate attributes to control
- * the allocation of the buffer. */
-PyAPI_FUNC(void)
-_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
-
-/* Prepare the buffer to write 'length' characters
- with the specified maximum character.
-
- Return 0 on success, raise an exception and return -1 on error. */
-#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \
- (((MAXCHAR) <= (WRITER)->maxchar \
- && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \
- ? 0 \
- : (((LENGTH) == 0) \
- ? 0 \
- : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR))))
-
-/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro
- instead. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
- Py_ssize_t length, Py_UCS4 maxchar);
-
-/* Prepare the buffer to have at least the kind KIND.
- For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will
- support characters in range U+000-U+FFFF.
-
- Return 0 on success, raise an exception and return -1 on error. */
-#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \
- (assert((KIND) != PyUnicode_WCHAR_KIND), \
- (KIND) <= (WRITER)->kind \
- ? 0 \
- : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND)))
-
-/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind()
- macro instead. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer,
- enum PyUnicode_Kind kind);
-
-/* Append a Unicode character.
- Return 0 on success, raise an exception and return -1 on error. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
- Py_UCS4 ch
- );
-
-/* Append a Unicode string.
- Return 0 on success, raise an exception and return -1 on error. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
- PyObject *str /* Unicode string */
- );
-
-/* Append a substring of a Unicode string.
- Return 0 on success, raise an exception and return -1 on error. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
- PyObject *str, /* Unicode string */
- Py_ssize_t start,
- Py_ssize_t end
- );
-
-/* Append an ASCII-encoded byte string.
- Return 0 on success, raise an exception and return -1 on error. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
- const char *str, /* ASCII-encoded byte string */
- Py_ssize_t len /* number of bytes, or -1 if unknown */
- );
-
-/* Append a latin1-encoded byte string.
- Return 0 on success, raise an exception and return -1 on error. */
-PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
- const char *str, /* latin1-encoded byte string */
- Py_ssize_t len /* length in bytes */
- );
-
-/* Get the value of the writer as a Unicode string. Clear the
- buffer of the writer. Raise an exception and return NULL
- on error. */
-PyAPI_FUNC(PyObject *)
-_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
-
-/* Deallocate memory of a writer (clear its internal buffer). */
-PyAPI_FUNC(void)
-_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
-#endif
-
-#ifndef Py_LIMITED_API
-/* Format the object based on the format_spec, as defined in PEP 3101
- (Advanced String Formatting). */
-PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter(
- _PyUnicodeWriter *writer,
- PyObject *obj,
- PyObject *format_spec,
- Py_ssize_t start,
- Py_ssize_t end);
-#endif
-
PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **);
PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **);
PyAPI_FUNC(PyObject *) PyUnicode_InternFromString(
const char *u /* UTF-8 encoded string */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void);
-#endif
/* Use only if you know it's a string */
#define PyUnicode_CHECK_INTERNED(op) \
@@ -1066,10 +315,6 @@ PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString(
Py_ssize_t *size /* number of characters of the result */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind);
-#endif
-
#endif
/* --- Unicode ordinals --------------------------------------------------- */
@@ -1114,60 +359,7 @@ PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
/* --- Manage the default encoding ---------------------------------------- */
-/* Returns a pointer to the default encoding (UTF-8) of the
- Unicode object unicode and the size of the encoded representation
- in bytes stored in *size.
-
- In case of an error, no *size is set.
-
- This function caches the UTF-8 encoded string in the unicodeobject
- and subsequent calls will return the same string. The memory is released
- when the unicodeobject is deallocated.
-
- _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to
- support the previous internal function with the same behaviour.
-
- *** This API is for interpreter INTERNAL USE ONLY and will likely
- *** be removed or changed in the future.
-
- *** If you need to access the Unicode object as UTF-8 bytes string,
- *** please use PyUnicode_AsUTF8String() instead.
-*/
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize(
- PyObject *unicode,
- Py_ssize_t *size);
-#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize
-#endif
-
-/* Returns a pointer to the default encoding (UTF-8) of the
- Unicode object unicode.
-
- Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation
- in the unicodeobject.
-
- _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to
- support the previous internal function with the same behaviour.
-
- Use of this API is DEPRECATED since no size information can be
- extracted from the returned data.
-
- *** This API is for interpreter INTERNAL USE ONLY and will likely
- *** be removed or changed for Python 3.1.
-
- *** If you need to access the Unicode object as UTF-8 bytes string,
- *** please use PyUnicode_AsUTF8String() instead.
-
-*/
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode);
-#define _PyUnicode_AsString PyUnicode_AsUTF8
-#endif
-
/* Returns "utf-8". */
-
PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
/* --- Generic Codecs ----------------------------------------------------- */
@@ -1208,18 +400,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedUnicode(
const char *errors /* error handling */
) Py_DEPRECATED(3.6);
-/* Encodes a Py_UNICODE buffer of the given size and returns a
- Python string object. */
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_Encode(
- const Py_UNICODE *s, /* Unicode char buffer */
- Py_ssize_t size, /* number of Py_UNICODE chars to encode */
- const char *encoding, /* encoding */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
/* Encodes a Unicode object and returns the result as Python
object.
@@ -1277,22 +457,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
Py_ssize_t *consumed /* bytes consumed */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* number of Py_UNICODE chars to encode */
- int base64SetO, /* Encode RFC2152 Set O characters in base64 */
- int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7(
- PyObject *unicode, /* Unicode object */
- int base64SetO, /* Encode RFC2152 Set O characters in base64 */
- int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */
- const char *errors /* error handling */
- );
-#endif
-
/* --- UTF-8 Codecs ------------------------------------------------------- */
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
@@ -1312,18 +476,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
- PyObject *unicode,
- const char *errors);
-
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* number of Py_UNICODE chars to encode */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
/* --- UTF-32 Codecs ------------------------------------------------------ */
/* Decodes length bytes from a UTF-32 encoded buffer string and returns
@@ -1391,20 +543,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
*/
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* number of Py_UNICODE chars to encode */
- const char *errors, /* error handling */
- int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
- ) Py_DEPRECATED(3.3);
-PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
- PyObject *object, /* Unicode object */
- const char *errors, /* error handling */
- int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
- );
-#endif
-
/* --- UTF-16 Codecs ------------------------------------------------------ */
/* Decodes length bytes from a UTF-16 encoded buffer string and returns
@@ -1456,40 +594,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
PyObject *unicode /* Unicode object */
);
-/* Returns a Python string object holding the UTF-16 encoded value of
- the Unicode data.
-
- If byteorder is not 0, output is written according to the following
- byte order:
-
- byteorder == -1: little endian
- byteorder == 0: native byte order (writes a BOM mark)
- byteorder == 1: big endian
-
- If byteorder is 0, the output string will always start with the
- Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
- prepended.
-
- Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
- UCS-2. This trick makes it possible to add full UTF-16 capabilities
- at a later point without compromising the APIs.
-
-*/
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* number of Py_UNICODE chars to encode */
- const char *errors, /* error handling */
- int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
- ) Py_DEPRECATED(3.3);
-PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(
- PyObject* unicode, /* Unicode object */
- const char *errors, /* error handling */
- int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */
- );
-#endif
-
/* --- Unicode-Escape Codecs ---------------------------------------------- */
PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
@@ -1498,30 +602,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
const char *errors /* error handling */
);
-#ifndef Py_LIMITED_API
-/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape
- chars. */
-PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape(
- const char *string, /* Unicode-Escape encoded string */
- Py_ssize_t length, /* size of string */
- const char *errors, /* error handling */
- const char **first_invalid_escape /* on return, points to first
- invalid escaped char in
- string. */
-);
-#endif
-
PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length /* Number of Py_UNICODE chars to encode */
- ) Py_DEPRECATED(3.3);
-#endif
-
/* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
@@ -1534,30 +618,9 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length /* Number of Py_UNICODE chars to encode */
- ) Py_DEPRECATED(3.3);
-#endif
-
-/* --- Unicode Internal Codec ---------------------------------------------
-
- Only for internal use in _codecsmodule.c */
-
-#ifndef Py_LIMITED_API
-PyObject *_PyUnicode_DecodeUnicodeInternal(
- const char *string,
- Py_ssize_t length,
- const char *errors
- );
-#endif
-
/* --- Latin-1 Codecs -----------------------------------------------------
- Note: Latin-1 corresponds to the first 256 Unicode ordinals.
-
-*/
+ Note: Latin-1 corresponds to the first 256 Unicode ordinals. */
PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
const char *string, /* Latin-1 encoded string */
@@ -1569,18 +632,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String(
- PyObject* unicode,
- const char* errors);
-
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
/* --- ASCII Codecs -------------------------------------------------------
Only 7-bit ASCII data is excepted. All other codes generate errors.
@@ -1597,18 +648,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString(
- PyObject* unicode,
- const char* errors);
-
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
/* --- Character Map Codecs -----------------------------------------------
This codec uses mappings to encode and decode characters.
@@ -1638,46 +677,9 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
PyObject *mapping /* encoding mapping */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
- PyObject *mapping, /* encoding mapping */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap(
- PyObject *unicode, /* Unicode object */
- PyObject *mapping, /* encoding mapping */
- const char *errors /* error handling */
- );
-#endif
-
-/* Translate a Py_UNICODE buffer of the given length by applying a
- character mapping table to it and return the resulting Unicode
- object.
-
- The mapping table must map Unicode ordinal integers to Unicode strings,
- Unicode ordinal integers or None (causing deletion of the character).
-
- Mapping tables may be dictionaries or sequences. Unmapped character
- ordinals (ones which cause a LookupError) are left untouched and
- are copied as-is.
-
-*/
-
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
- PyObject *table, /* Translate table */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
-#ifdef MS_WINDOWS
-
/* --- MBCS codecs for Windows -------------------------------------------- */
+#ifdef MS_WINDOWS
PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
const char *string, /* MBCS encoded string */
Py_ssize_t length, /* size of string */
@@ -1705,14 +707,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
PyObject *unicode /* Unicode object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
- const Py_UNICODE *data, /* Unicode char buffer */
- Py_ssize_t length, /* number of Py_UNICODE chars to encode */
- const char *errors /* error handling */
- ) Py_DEPRECATED(3.3);
-#endif
-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage(
int code_page, /* code page number */
@@ -1723,61 +717,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage(
#endif /* MS_WINDOWS */
-#ifndef Py_LIMITED_API
-/* --- Decimal Encoder ---------------------------------------------------- */
-
-/* Takes a Unicode string holding a decimal value and writes it into
- an output buffer using standard ASCII digit codes.
-
- The output buffer has to provide at least length+1 bytes of storage
- area. The output string is 0-terminated.
-
- The encoder converts whitespace to ' ', decimal characters to their
- corresponding ASCII digit and all other Latin-1 characters except
- \0 as-is. Characters outside this range (Unicode ordinals 1-256)
- are treated as errors. This includes embedded NULL bytes.
-
- Error handling is defined by the errors argument:
-
- NULL or "strict": raise a ValueError
- "ignore": ignore the wrong characters (these are not copied to the
- output buffer)
- "replace": replaces illegal characters with '?'
-
- Returns 0 on success, -1 on failure.
-
-*/
-
-PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
- Py_UNICODE *s, /* Unicode buffer */
- Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
- char *output, /* Output buffer; must have size >= length */
- const char *errors /* error handling */
- ) /* Py_DEPRECATED(3.3) */;
-
-/* Transforms code points that have decimal digit property to the
- corresponding ASCII digit code points.
-
- Returns a new Unicode string on success, NULL on failure.
-*/
-
-PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII(
- Py_UNICODE *s, /* Unicode buffer */
- Py_ssize_t length /* Number of Py_UNICODE chars to transform */
- ) /* Py_DEPRECATED(3.3) */;
-
-/* Coverts a Unicode object holding a decimal value to an ASCII string
- for using in int, float and complex parsers.
- Transforms code points that have decimal digit property to the
- corresponding ASCII digit code points. Transforms spaces to ASCII.
- Transforms code points starting from the first non-ASCII code point that
- is neither a decimal digit nor a space to the end into '?'. */
-
-PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
- PyObject *unicode /* Unicode object */
- );
-#endif
-
/* --- Locale encoding --------------------------------------------------- */
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
@@ -1976,14 +915,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_Join(
PyObject *seq /* Sequence object */
);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyUnicode_JoinArray(
- PyObject *separator,
- PyObject *const *items,
- Py_ssize_t seqlen
- );
-#endif /* Py_LIMITED_API */
-
/* Return 1 if substr matches str[start:end] at the given tail end, 0
otherwise. */
@@ -2047,17 +978,6 @@ PyAPI_FUNC(int) PyUnicode_Compare(
PyObject *right /* Right string */
);
-#ifndef Py_LIMITED_API
-/* Test whether a unicode is equal to ASCII identifier. Return 1 if true,
- 0 otherwise. The right argument must be ASCII identifier.
- Any error occurs inside will be cleared before return. */
-
-PyAPI_FUNC(int) _PyUnicode_EqualToASCIIId(
- PyObject *left, /* Left string */
- _Py_Identifier *right /* Right identifier */
- );
-#endif
-
/* Compare a Unicode object with C string and return -1, 0, 1 for less than,
equal, and greater than, respectively. It is best to pass only
ASCII-encoded strings, but the function interprets the input string as
@@ -2069,17 +989,6 @@ PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString(
const char *right /* ASCII-encoded string */
);
-#ifndef Py_LIMITED_API
-/* Test whether a unicode is equal to ASCII string. Return 1 if true,
- 0 otherwise. The right argument must be ASCII-encoded string.
- Any error occurs inside will be cleared before return. */
-
-PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(
- PyObject *left,
- const char *right /* ASCII-encoded string */
- );
-#endif
-
/* Rich compare two strings and return one of the following:
- NULL in case an exception was raised
@@ -2121,192 +1030,8 @@ PyAPI_FUNC(int) PyUnicode_Contains(
PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);
-#ifndef Py_LIMITED_API
-/* Externally visible for str.strip(unicode) */
-PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
- PyObject *self,
- int striptype,
- PyObject *sepobj
- );
-#endif
-
-/* Using explicit passed-in values, insert the thousands grouping
- into the string pointed to by buffer. For the argument descriptions,
- see Objects/stringlib/localeutil.h */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
- _PyUnicodeWriter *writer,
- Py_ssize_t n_buffer,
- PyObject *digits,
- Py_ssize_t d_pos,
- Py_ssize_t n_digits,
- Py_ssize_t min_width,
- const char *grouping,
- PyObject *thousands_sep,
- Py_UCS4 *maxchar);
-#endif
/* === Characters Type APIs =============================================== */
-/* Helper array used by Py_UNICODE_ISSPACE(). */
-
-#ifndef Py_LIMITED_API
-PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
-
-/* These should not be used directly. Use the Py_UNICODE_IS* and
- Py_UNICODE_TO* macros instead.
-
- These APIs are implemented in Objects/unicodectype.c.
-
-*/
-
-PyAPI_FUNC(int) _PyUnicode_IsLowercase(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsUppercase(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsXidStart(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsXidContinue(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
- const Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
- const Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase(
- Py_UCS4 ch /* Unicode character */
- ) /* Py_DEPRECATED(3.3) */;
-
-PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase(
- Py_UCS4 ch /* Unicode character */
- ) /* Py_DEPRECATED(3.3) */;
-
-PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase(
- Py_UCS4 ch /* Unicode character */
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(int) _PyUnicode_ToLowerFull(
- Py_UCS4 ch, /* Unicode character */
- Py_UCS4 *res
- );
-
-PyAPI_FUNC(int) _PyUnicode_ToTitleFull(
- Py_UCS4 ch, /* Unicode character */
- Py_UCS4 *res
- );
-
-PyAPI_FUNC(int) _PyUnicode_ToUpperFull(
- Py_UCS4 ch, /* Unicode character */
- Py_UCS4 *res
- );
-
-PyAPI_FUNC(int) _PyUnicode_ToFoldedFull(
- Py_UCS4 ch, /* Unicode character */
- Py_UCS4 *res
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsCased(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_ToDigit(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(double) _PyUnicode_ToNumeric(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsDigit(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsNumeric(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsPrintable(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(int) _PyUnicode_IsAlpha(
- Py_UCS4 ch /* Unicode character */
- );
-
-PyAPI_FUNC(size_t) Py_UNICODE_strlen(
- const Py_UNICODE *u
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcpy(
- Py_UNICODE *s1,
- const Py_UNICODE *s2) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcat(
- Py_UNICODE *s1, const Py_UNICODE *s2) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strncpy(
- Py_UNICODE *s1,
- const Py_UNICODE *s2,
- size_t n) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(int) Py_UNICODE_strcmp(
- const Py_UNICODE *s1,
- const Py_UNICODE *s2
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(int) Py_UNICODE_strncmp(
- const Py_UNICODE *s1,
- const Py_UNICODE *s2,
- size_t n
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strchr(
- const Py_UNICODE *s,
- Py_UNICODE c
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strrchr(
- const Py_UNICODE *s,
- Py_UNICODE c
- ) Py_DEPRECATED(3.3);
-
-PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int);
-
-/* Create a copy of a unicode string ending with a nul character. Return NULL
- and raise a MemoryError exception on memory allocation failure, otherwise
- return a new allocated buffer (use PyMem_Free() to free the buffer). */
-
-PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy(
- PyObject *unicode
- ) Py_DEPRECATED(3.3);
-#endif /* Py_LIMITED_API */
-
#if defined(Py_DEBUG) && !defined(Py_LIMITED_API)
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
PyObject *op,
@@ -2318,15 +1043,10 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
#endif
#ifndef Py_LIMITED_API
-/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/
-PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
-/* Clear all static strings. */
-PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void);
-
-/* Fast equality check when the inputs are known to be exact unicode types
- and where the hash values are equal (i.e. a very probable match) */
-PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *);
-#endif /* !Py_LIMITED_API */
+# define Py_CPYTHON_UNICODEOBJECT_H
+# include "cpython/unicodeobject.h"
+# undef Py_CPYTHON_UNICODEOBJECT_H
+#endif
#ifdef __cplusplus
}
More information about the Python-checkins
mailing list