/*

Unicode implementation based on original code by Fredrik Lundh,

modified by Marc-Andre Lemburg <mal@lemburg.com> according to the

Unicode Integration Proposal (see file Misc/unicode.txt).

Copyright (c) Corporation for National Research Initiatives.

--------------------------------------------------------------------

The original string type implementation is:

Copyright (c) 1999 by Secret Labs AB

Copyright (c) 1999 by Fredrik Lundh

By obtaining, using, and/or copying this software and/or its

associated documentation, you agree that you have read, understood,

and will comply with the following terms and conditions:

Permission to use, copy, modify, and distribute this software and its

associated documentation for any purpose and without fee is hereby

granted, provided that the above copyright notice appears in all

copies, and that both that copyright notice and this permission notice

appear in supporting documentation, and that the name of Secret Labs

AB or the author not be used in advertising or publicity pertaining to

distribution of the software without specific, written prior

permission.

SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO

THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND

FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR

ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN

ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT

OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

--------------------------------------------------------------------

*/

#include "Python.h"

#include "unicodeobject.h"

#include "ucnhash.h"

#ifdef MS_WIN32

#include <windows.h>

#endif

/* Limit for the Unicode object free list */

#define MAX_UNICODE_FREELIST_SIZE 1024

/* Limit for the Unicode object free list stay alive optimization.

The implementation will keep allocated Unicode memory intact for

all objects on the free list having a size less than this

limit. This reduces malloc() overhead for small Unicode objects.

At worst this will result in MAX_UNICODE_FREELIST_SIZE *

(sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +

malloc()-overhead) bytes of unused garbage.

Setting the limit to 0 effectively turns the feature off.

Note: This is an experimental feature ! If you get core dumps when

using Unicode objects, turn this feature off.

*/

#define KEEPALIVE_SIZE_LIMIT 9

/* Endianness switches; defaults to little endian */

#ifdef WORDS_BIGENDIAN

# define BYTEORDER_IS_BIG_ENDIAN

#else

# define BYTEORDER_IS_LITTLE_ENDIAN

#endif

/* --- Globals ------------------------------------------------------------

The globals are initialized by the _PyUnicode_Init() API and should

not be used before calling that API.

*/

/* The empty Unicode object */

static PyUnicodeObject *unicode_empty;

/* Free list for Unicode objects */

static PyUnicodeObject *unicode_freelist;

static int unicode_freelist_size;

/* Default encoding to use and assume when NULL is passed as encoding

parameter; it is initialized by _PyUnicode_Init().

Always use the PyUnicode_SetDefaultEncoding() and

PyUnicode_GetDefaultEncoding() APIs to access this global.

*/

static char unicode_default_encoding[100];

/* --- Unicode Object ----------------------------------------------------- */

static

int _PyUnicode_Resize(register PyUnicodeObject *unicode,

int length)

{

void *oldstr;

/* Shortcut if there's nothing much to do. */

if (unicode->length == length)

goto reset;

/* Resizing unicode_empty is not allowed. */

if (unicode == unicode_empty) {

PyErr_SetString(PyExc_SystemError,

"can't resize empty unicode object");

return -1;

}

/* We allocate one more byte to make sure the string is

Ux0000 terminated -- XXX is this needed ? */

oldstr = unicode->str;

PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);

if (!unicode->str) {

unicode->str = oldstr;

PyErr_NoMemory();

return -1;

}

unicode->str[length] = 0;

unicode->length = length;

reset:

/* Reset the object caches */

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

unicode->hash = -1;

return 0;

}

int PyUnicode_Resize(PyObject **unicode,

int length)

{

PyUnicodeObject *v;

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

v = (PyUnicodeObject *)*unicode;

if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1) {

PyErr_BadInternalCall();

return -1;

}

return _PyUnicode_Resize(v, length);

}

/* We allocate one more byte to make sure the string is

Ux0000 terminated -- XXX is this needed ?

XXX This allocator could further be enhanced by assuring that the

free list never reduces its size below 1.

*/

static

PyUnicodeObject *_PyUnicode_New(int length)

{

register PyUnicodeObject *unicode;

/* Optimization for empty strings */

if (length == 0 && unicode_empty != NULL) {

Py_INCREF(unicode_empty);

return unicode_empty;

}

/* Unicode freelist & memory allocation */

if (unicode_freelist) {

unicode = unicode_freelist;

unicode_freelist = *(PyUnicodeObject **)unicode;

unicode_freelist_size--;

if (unicode->str) {

/* Keep-Alive optimization: we only upsize the buffer,

never downsize it. */

if ((unicode->length < length) &&

_PyUnicode_Resize(unicode, length)) {

PyMem_DEL(unicode->str);

goto onError;

}

}

else {

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

PyObject_INIT(unicode, &PyUnicode_Type);

}

else {

unicode = PyObject_NEW(PyUnicodeObject, &PyUnicode_Type);

if (unicode == NULL)

return NULL;

unicode->str = PyMem_NEW(Py_UNICODE, length + 1);

}

if (!unicode->str) {

PyErr_NoMemory();

goto onError;

}

unicode->str[length] = 0;

unicode->length = length;

unicode->hash = -1;

unicode->defenc = NULL;

return unicode;

onError:

_Py_ForgetReference((PyObject *)unicode);

PyObject_DEL(unicode);

return NULL;

}

static

void _PyUnicode_Free(register PyUnicodeObject *unicode)

{

if (unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {

/* Keep-Alive optimization */

if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {

PyMem_DEL(unicode->str);

unicode->str = NULL;

unicode->length = 0;

}

if (unicode->defenc) {

Py_DECREF(unicode->defenc);

unicode->defenc = NULL;

}

/* Add to free list */

*(PyUnicodeObject **)unicode = unicode_freelist;

unicode_freelist = unicode;

unicode_freelist_size++;

}

else {

PyMem_DEL(unicode->str);

Py_XDECREF(unicode->defenc);

PyObject_DEL(unicode);

}

}

PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,

int size)

{

PyUnicodeObject *unicode;

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the Unicode data into the new object */

if (u != NULL)

memcpy(unicode->str, u, size * sizeof(Py_UNICODE));

return (PyObject *)unicode;

}

#ifdef HAVE_WCHAR_H

PyObject *PyUnicode_FromWideChar(register const wchar_t *w,

int size)

{

PyUnicodeObject *unicode;

if (w == NULL) {

PyErr_BadInternalCall();

return NULL;

}

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

/* Copy the wchar_t data into the new object */

#ifdef HAVE_USABLE_WCHAR_T

memcpy(unicode->str, w, size * sizeof(wchar_t));

#else

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*u++ = *w++;

}

#endif

return (PyObject *)unicode;

}

int PyUnicode_AsWideChar(PyUnicodeObject *unicode,

register wchar_t *w,

int size)

{

if (unicode == NULL) {

PyErr_BadInternalCall();

return -1;

}

if (size > PyUnicode_GET_SIZE(unicode))

size = PyUnicode_GET_SIZE(unicode);

#ifdef HAVE_USABLE_WCHAR_T

memcpy(w, unicode->str, size * sizeof(wchar_t));

#else

{

register Py_UNICODE *u;

register int i;

u = PyUnicode_AS_UNICODE(unicode);

for (i = size; i >= 0; i--)

*w++ = *u++;

}

#endif

return size;

}

#endif

PyObject *PyUnicode_FromObject(register PyObject *obj)

{

return PyUnicode_FromEncodedObject(obj, NULL, "strict");

}

PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,

const char *encoding,

const char *errors)

{

const char *s;

int len;

int owned = 0;

PyObject *v;

if (obj == NULL) {

PyErr_BadInternalCall();

return NULL;

}

/* Coerce object */

if (PyInstance_Check(obj)) {

PyObject *func;

func = PyObject_GetAttrString(obj, "__str__");

if (func == NULL) {

PyErr_SetString(PyExc_TypeError,

"coercing to Unicode: instance doesn't define __str__");

return NULL;

}

obj = PyEval_CallObject(func, NULL);

Py_DECREF(func);

if (obj == NULL)

return NULL;

owned = 1;

}

if (PyUnicode_Check(obj)) {

Py_INCREF(obj);

v = obj;

if (encoding) {

PyErr_SetString(PyExc_TypeError,

"decoding Unicode is not supported");

return NULL;

}

goto done;

}

else if (PyString_Check(obj)) {

s = PyString_AS_STRING(obj);

len = PyString_GET_SIZE(obj);

}

else if (PyObject_AsCharBuffer(obj, &s, &len)) {

/* Overwrite the error message with something more useful in

case of a TypeError. */

if (PyErr_ExceptionMatches(PyExc_TypeError))

PyErr_Format(PyExc_TypeError,

"coercing to Unicode: need string or buffer, "

"%.80s found",

obj->ob_type->tp_name);

goto onError;

}

/* Convert to Unicode */

if (len == 0) {

Py_INCREF(unicode_empty);

v = (PyObject *)unicode_empty;

}

else

v = PyUnicode_Decode(s, len, encoding, errors);

done:

if (owned) {

Py_DECREF(obj);

}

return v;

onError:

if (owned) {

Py_DECREF(obj);

}

return NULL;

}

PyObject *PyUnicode_Decode(const char *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *buffer = NULL, *unicode;

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_DecodeUTF8(s, size, errors);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_DecodeLatin1(s, size, errors);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_DecodeASCII(s, size, errors);

/* Decode via the codec registry */

buffer = PyBuffer_FromMemory((void *)s, size);

if (buffer == NULL)

goto onError;

unicode = PyCodec_Decode(buffer, encoding, errors);

if (unicode == NULL)

goto onError;

if (!PyUnicode_Check(unicode)) {

PyErr_Format(PyExc_TypeError,

"decoder did not return an unicode object (type=%.400s)",

unicode->ob_type->tp_name);

Py_DECREF(unicode);

goto onError;

}

Py_DECREF(buffer);

return unicode;

onError:

Py_XDECREF(buffer);

return NULL;

}

PyObject *PyUnicode_Encode(const Py_UNICODE *s,

int size,

const char *encoding,

const char *errors)

{

PyObject *v, *unicode;

unicode = PyUnicode_FromUnicode(s, size);

if (unicode == NULL)

return NULL;

v = PyUnicode_AsEncodedString(unicode, encoding, errors);

Py_DECREF(unicode);

return v;

}

PyObject *PyUnicode_AsEncodedString(PyObject *unicode,

const char *encoding,

const char *errors)

{

PyObject *v;

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

if (encoding == NULL)

encoding = PyUnicode_GetDefaultEncoding();

/* Shortcuts for common default encodings */

if (errors == NULL) {

if (strcmp(encoding, "utf-8") == 0)

return PyUnicode_AsUTF8String(unicode);

else if (strcmp(encoding, "latin-1") == 0)

return PyUnicode_AsLatin1String(unicode);

else if (strcmp(encoding, "ascii") == 0)

return PyUnicode_AsASCIIString(unicode);

}

/* Encode via the codec registry */

v = PyCodec_Encode(unicode, encoding, errors);

if (v == NULL)

goto onError;

/* XXX Should we really enforce this ? */

if (!PyString_Check(v)) {

PyErr_Format(PyExc_TypeError,

"encoder did not return a string object (type=%.400s)",

v->ob_type->tp_name);

Py_DECREF(v);

goto onError;

}

return v;

onError:

return NULL;

}

/* Return a Python string holding the default encoded value of the

Unicode object.

The resulting string is cached in the Unicode object for subsequent

usage by this function. The cached version is needed to implement

the character buffer interface and will live (at least) as long as

the Unicode object itself.

The refcount of the string is *not* incremented.

*** Exported for internal use by the interpreter only !!! ***

*/

PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,

const char *errors)

{

PyObject *v = ((PyUnicodeObject *)unicode)->defenc;

if (v)

return v;

v = PyUnicode_AsEncodedString(unicode, NULL, errors);

if (v && errors == NULL)

((PyUnicodeObject *)unicode)->defenc = v;

return v;

}

Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_AS_UNICODE(unicode);

onError:

return NULL;

}

int PyUnicode_GetSize(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

goto onError;

}

return PyUnicode_GET_SIZE(unicode);

onError:

return -1;

}

const char *PyUnicode_GetDefaultEncoding(void)

{

return unicode_default_encoding;

}

int PyUnicode_SetDefaultEncoding(const char *encoding)

{

PyObject *v;

/* Make sure the encoding is valid. As side effect, this also

loads the encoding into the codec registry cache. */

v = _PyCodec_Lookup(encoding);

if (v == NULL)

goto onError;

Py_DECREF(v);

strncpy(unicode_default_encoding,

encoding,

sizeof(unicode_default_encoding));

return 0;

onError:

return -1;

}

/* --- UTF-8 Codec -------------------------------------------------------- */

static

char utf8_code_length[256] = {

/* Map UTF-8 encoded prefix byte to sequence length. zero means

illegal prefix. see RFC 2279 for details */

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,

2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,

3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,

4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0

};

static

int utf8_decoding_error(const char **source,

Py_UNICODE **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-8 decoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

(*source)++;

return 0;

}

else if (strcmp(errors,"replace") == 0) {

(*source)++;

**dest = Py_UNICODE_REPLACEMENT_CHARACTER;

(*dest)++;

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-8 decoding error; unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_DecodeUTF8(const char *s,

int size,

const char *errors)

{

int n;

const char *e;

PyUnicodeObject *unicode;

Py_UNICODE *p;

const char *errmsg = "";

/* Note: size will always be longer than the resulting Unicode

character count */

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

if (size == 0)

return (PyObject *)unicode;

/* Unpack UTF-8 encoded data */

p = unicode->str;

e = s + size;

while (s < e) {

Py_UCS4 ch = (unsigned char)*s;

if (ch < 0x80) {

*p++ = (Py_UNICODE)ch;

s++;

continue;

}

n = utf8_code_length[ch];

if (s + n > e) {

errmsg = "unexpected end of data";

goto utf8Error;

}

switch (n) {

case 0:

errmsg = "unexpected code byte";

goto utf8Error;

break;

case 1:

errmsg = "internal error";

goto utf8Error;

break;

case 2:

if ((s[1] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);

if (ch < 0x80) {

errmsg = "illegal encoding";

goto utf8Error;

}

else

*p++ = (Py_UNICODE)ch;

break;

case 3:

if ((s[1] & 0xc0) != 0x80 ||

(s[2] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);

if (ch < 0x800 || (ch >= 0xd800 && ch < 0xe000)) {

errmsg = "illegal encoding";

goto utf8Error;

}

else

*p++ = (Py_UNICODE)ch;

break;

case 4:

if ((s[1] & 0xc0) != 0x80 ||

(s[2] & 0xc0) != 0x80 ||

(s[3] & 0xc0) != 0x80) {

errmsg = "invalid data";

goto utf8Error;

}

ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +

((s[2] & 0x3f) << 6) + (s[3] & 0x3f);

/* validate and convert to UTF-16 */

if ((ch < 0x10000) || /* minimum value allowed for 4

byte encoding */

(ch > 0x10ffff)) { /* maximum value allowed for

UTF-16 */

errmsg = "illegal encoding";

goto utf8Error;

}

/* compute and append the two surrogates: */

/* translate from 10000..10FFFF to 0..FFFF */

ch -= 0x10000;

/* high surrogate = top 10 bits added to D800 */

*p++ = (Py_UNICODE)(0xD800 + (ch >> 10));

/* low surrogate = bottom 10 bits added to DC00 */

*p++ = (Py_UNICODE)(0xDC00 + (ch & ~0xFC00));

break;

default:

/* Other sizes are only needed for UCS-4 */

errmsg = "unsupported Unicode code range";

goto utf8Error;

break;

}

s += n;

continue;

utf8Error:

if (utf8_decoding_error(&s, &p, errors, errmsg))

goto onError;

}

/* Adjust length */

if (_PyUnicode_Resize(unicode, p - unicode->str))

goto onError;

return (PyObject *)unicode;

onError:

Py_DECREF(unicode);

return NULL;

}

/* Not used anymore, now that the encoder supports UTF-16

surrogates. */

#if 0

static

int utf8_encoding_error(const Py_UNICODE **source,

char **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-8 encoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

return 0;

}

else if (strcmp(errors,"replace") == 0) {

**dest = '?';

(*dest)++;

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-8 encoding error; "

"unknown error handling code: %.400s",

errors);

return -1;

}

}

#endif

PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,

int size,

const char *errors)

{

PyObject *v;

char *p;

char *q;

Py_UCS4 ch2;

unsigned int cbAllocated = 3 * size;

unsigned int cbWritten = 0;

int i = 0;

v = PyString_FromStringAndSize(NULL, cbAllocated);

if (v == NULL)

return NULL;

if (size == 0)

return v;

p = q = PyString_AS_STRING(v);

while (i < size) {

Py_UCS4 ch = s[i++];

if (ch < 0x80) {

*p++ = (char) ch;

cbWritten++;

}

else if (ch < 0x0800) {

*p++ = 0xc0 | (ch >> 6);

*p++ = 0x80 | (ch & 0x3f);

cbWritten += 2;

}

else {

/* Check for high surrogate */

if (0xD800 <= ch && ch <= 0xDBFF) {

if (i != size) {

ch2 = s[i];

if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {

if (cbWritten >= (cbAllocated - 4)) {

/* Provide enough room for some more

surrogates */

cbAllocated += 4*10;

if (_PyString_Resize(&v, cbAllocated))

goto onError;

}

/* combine the two values */

ch = ((ch - 0xD800)<<10 | (ch2-0xDC00))+0x10000;

*p++ = (char)((ch >> 18) | 0xf0);

*p++ = (char)(0x80 | ((ch >> 12) & 0x3f));

i++;

cbWritten += 4;

}

}

}

else {

*p++ = (char)(0xe0 | (ch >> 12));

cbWritten += 3;

}

*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

*p++ = (char)(0x80 | (ch & 0x3f));

}

}

*p = '\0';

if (_PyString_Resize(&v, p - q))

goto onError;

return v;

onError:

return NULL;

}

PyObject *PyUnicode_AsUTF8String(PyObject *unicode)

{

if (!PyUnicode_Check(unicode)) {

PyErr_BadArgument();

return NULL;

}

return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),

PyUnicode_GET_SIZE(unicode),

NULL);

}

/* --- UTF-16 Codec ------------------------------------------------------- */

static

int utf16_decoding_error(const Py_UNICODE **source,

Py_UNICODE **dest,

const char *errors,

const char *details)

{

if ((errors == NULL) ||

(strcmp(errors,"strict") == 0)) {

PyErr_Format(PyExc_UnicodeError,

"UTF-16 decoding error: %.400s",

details);

return -1;

}

else if (strcmp(errors,"ignore") == 0) {

return 0;

}

else if (strcmp(errors,"replace") == 0) {

if (dest) {

**dest = Py_UNICODE_REPLACEMENT_CHARACTER;

(*dest)++;

}

return 0;

}

else {

PyErr_Format(PyExc_ValueError,

"UTF-16 decoding error; "

"unknown error handling code: %.400s",

errors);

return -1;

}

}

PyObject *PyUnicode_DecodeUTF16(const char *s,

int size,

const char *errors,

int *byteorder)

{

PyUnicodeObject *unicode;

Py_UNICODE *p;

const Py_UNICODE *q, *e;

int bo = 0;

const char *errmsg = "";

/* size should be an even number */

if (size % sizeof(Py_UNICODE) != 0) {

if (utf16_decoding_error(NULL, NULL, errors, "truncated data"))

return NULL;

/* The remaining input chars are ignored if we fall through

here... */

}

/* Note: size will always be longer than the resulting Unicode

character count */

unicode = _PyUnicode_New(size);

if (!unicode)

return NULL;

if (size == 0)

return (PyObject *)unicode;

/* Unpack UTF-16 encoded data */

p = unicode->str;

q = (Py_UNICODE *)s;

e = q + (size / sizeof(Py_UNICODE));

if (byteorder)

bo = *byteorder;

while (q < e) {

register Py_UNICODE ch = *q++;

/* Check for BOM marks (U+FEFF) in the input and adjust

current byte order setting accordingly. Swap input

bytes if needed. (This assumes sizeof(Py_UNICODE) == 2

!) */

#ifdef BYTEORDER_IS_LITTLE_ENDIAN

if (ch == 0xFEFF) {

bo = -1;

continue;

} else if (ch == 0xFFFE) {

bo = 1;

continue;

}

if (bo == 1)

ch = (ch >> 8) | (ch << 8);

#else

if (ch == 0xFEFF) {

bo = 1;

continue;

} else if (ch == 0xFFFE) {

bo = -1;

continue;

}

if (bo == -1)

ch = (ch >> 8) | (ch << 8);

#endif

if (ch < 0xD800 || ch > 0xDFFF) {

*p++ = ch;

continue;

}

/* UTF-16 code pair: */

if (q >= e) {

errmsg = "unexpected end of data";

goto utf16Error;

}

if (0xDC00 <= *q && *q <= 0xDFFF) {

q++;

if (0xD800 <= *q && *q <= 0xDBFF) {

/* This is valid data (a UTF-16 surrogate pair), but

we are not able to store this information since our

Py_UNICODE type only has 16 bits... this might

change someday, even though it's unlikely. */

errmsg = "code pairs are not supported";

goto utf16Error;

}

else

continue;

}