/*

* multibytecodec.c: Common Multibyte Codec Implementation

*

* Written by Hye-Shik Chang <perky@FreeBSD.org>

* $CJKCodecs: multibytecodec.c,v 1.13 2004/08/19 16:57:19 perky Exp $

*/

#include "Python.h"

#include "multibytecodec.h"

typedef struct {

const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;

unsigned char *outbuf, *outbuf_end;

PyObject *excobj, *outobj;

} MultibyteEncodeBuffer;

typedef struct {

const unsigned char *inbuf, *inbuf_top, *inbuf_end;

Py_UNICODE *outbuf, *outbuf_end;

PyObject *excobj, *outobj;

} MultibyteDecodeBuffer;

PyDoc_STRVAR(MultibyteCodec_Encode__doc__,

"I.encode(unicode[, errors]) -> (string, length consumed)\n\

\n\

Return an encoded string version of `unicode'. errors may be given to\n\

set a different error handling scheme. Default is 'strict' meaning that\n\

encoding errors raise a UnicodeEncodeError. Other possible values are\n\

'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\

registered with codecs.register_error that can handle UnicodeEncodeErrors.");

PyDoc_STRVAR(MultibyteCodec_Decode__doc__,

"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\

\n\

Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\

to set a different error handling scheme. Default is 'strict' meaning\n\

that encoding errors raise a UnicodeDecodeError. Other possible values\n\

are 'ignore' and 'replace' as well as any other name registerd with\n\

codecs.register_error that is able to handle UnicodeDecodeErrors.");

PyDoc_STRVAR(MultibyteCodec_StreamReader__doc__,

"I.StreamReader(stream[, errors]) -> StreamReader instance");

PyDoc_STRVAR(MultibyteCodec_StreamWriter__doc__,

"I.StreamWriter(stream[, errors]) -> StreamWriter instance");

static char *codeckwarglist[] = {"input", "errors", NULL};

static char *streamkwarglist[] = {"stream", "errors", NULL};

static PyObject *multibytecodec_encode(MultibyteCodec *,

MultibyteCodec_State *, const Py_UNICODE **, size_t,

PyObject *, int);

static PyObject *mbstreamreader_create(MultibyteCodec *,

PyObject *, const char *);

static PyObject *mbstreamwriter_create(MultibyteCodec *,

PyObject *, const char *);

#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */

static PyObject *

make_tuple(PyObject *unicode, int len)

{

PyObject *v, *w;

if (unicode == NULL)

return NULL;

v = PyTuple_New(2);

if (v == NULL) {

Py_DECREF(unicode);

return NULL;

}

PyTuple_SET_ITEM(v, 0, unicode);

w = PyInt_FromLong(len);

if (w == NULL) {

Py_DECREF(v);

return NULL;

}

PyTuple_SET_ITEM(v, 1, w);

return v;

}

static PyObject *

get_errorcallback(const char *errors)

{

if (errors == NULL || strcmp(errors, "strict") == 0)

return ERROR_STRICT;

else if (strcmp(errors, "ignore") == 0)

return ERROR_IGNORE;

else if (strcmp(errors, "replace") == 0)

return ERROR_REPLACE;

else {

return PyCodec_LookupError(errors);

}

}

static int

expand_encodebuffer(MultibyteEncodeBuffer *buf, int esize)

{

int orgpos, orgsize, incsize;

orgpos = (int)((char*)buf->outbuf - PyString_AS_STRING(buf->outobj));

orgsize = PyString_GET_SIZE(buf->outobj);

incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);

if (orgsize > INT_MAX - incsize)

return -1;

if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)

return -1;

buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;

buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)

+ PyString_GET_SIZE(buf->outobj);

return 0;

}

#define REQUIRE_ENCODEBUFFER(buf, s) { \

if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \

if (expand_encodebuffer(buf, s) == -1) \

goto errorexit; \

}

static int

expand_decodebuffer(MultibyteDecodeBuffer *buf, int esize)

{

int orgpos, orgsize;

orgpos = (int)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));

orgsize = PyUnicode_GET_SIZE(buf->outobj);

if (PyUnicode_Resize(&buf->outobj, orgsize + (

esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)

return -1;

buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;

buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)

+ PyUnicode_GET_SIZE(buf->outobj);

return 0;

}

#define REQUIRE_DECODEBUFFER(buf, s) { \

if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \

if (expand_decodebuffer(buf, s) == -1) \

goto errorexit; \

}

static int

multibytecodec_encerror(MultibyteCodec *codec,

MultibyteCodec_State *state,

MultibyteEncodeBuffer *buf,

PyObject *errors, int e)

{

PyObject *retobj = NULL, *retstr = NULL, *argsobj, *tobj;

int retstrsize, newpos;

const char *reason;

size_t esize;

int start, end;

if (e > 0) {

reason = "illegal multibyte sequence";

esize = e;

}

else {

switch (e) {

case MBERR_TOOSMALL:

REQUIRE_ENCODEBUFFER(buf, -1);

return 0; /* retry it */

case MBERR_TOOFEW:

reason = "incomplete multibyte sequence";

esize = (size_t)(buf->inbuf_end - buf->inbuf);

break;

case MBERR_INTERNAL:

PyErr_SetString(PyExc_RuntimeError,

"internal codec error");

return -1;

default:

PyErr_SetString(PyExc_RuntimeError,

"unknown runtime error");

return -1;

}

}

if (errors == ERROR_REPLACE) {

const Py_UNICODE replchar = '?', *inbuf = &replchar;

int r;

for (;;) {

size_t outleft;

outleft = (size_t)(buf->outbuf_end - buf->outbuf);

r = codec->encode(state, codec->config, &inbuf, 1,

&buf->outbuf, outleft, 0);

if (r == MBERR_TOOSMALL) {

REQUIRE_ENCODEBUFFER(buf, -1);

continue;

}

else

break;

}

if (r != 0) {

REQUIRE_ENCODEBUFFER(buf, 1);

*buf->outbuf++ = '?';

}

}

if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

buf->inbuf += esize;

return 0;

}

start = (int)(buf->inbuf - buf->inbuf_top);

end = start + esize;

/* use cached exception object if available */

if (buf->excobj == NULL) {

buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,

buf->inbuf_top,

(int)(buf->inbuf_end - buf->inbuf_top),

start, end, reason);

if (buf->excobj == NULL)

goto errorexit;

}

else

if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||

PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||

PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)

goto errorexit;

if (errors == ERROR_STRICT) {

PyCodec_StrictErrors(buf->excobj);

goto errorexit;

}

argsobj = PyTuple_New(1);

if (argsobj == NULL)

goto errorexit;

PyTuple_SET_ITEM(argsobj, 0, buf->excobj);

Py_INCREF(buf->excobj);

retobj = PyObject_CallObject(errors, argsobj);

Py_DECREF(argsobj);

if (retobj == NULL)

goto errorexit;

if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||

!PyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {

PyErr_SetString(PyExc_ValueError,

"encoding error handler must return "

"(unicode, int) tuple");

goto errorexit;

}

{

const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);

retstr = multibytecodec_encode(codec, state, &uraw,

PyUnicode_GET_SIZE(tobj), ERROR_STRICT,

MBENC_FLUSH);

if (retstr == NULL)

goto errorexit;

}

retstrsize = PyString_GET_SIZE(retstr);

REQUIRE_ENCODEBUFFER(buf, retstrsize);

memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);

buf->outbuf += retstrsize;

newpos = (int)PyInt_AS_LONG(PyTuple_GET_ITEM(retobj, 1));

if (newpos < 0)

newpos += (int)(buf->inbuf_end - buf->inbuf_top);

if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

PyErr_Format(PyExc_IndexError,

"position %d from error handler out of bounds",

newpos);

goto errorexit;

}

buf->inbuf = buf->inbuf_top + newpos;

Py_DECREF(retobj);

Py_DECREF(retstr);

return 0;

errorexit:

Py_XDECREF(retobj);

Py_XDECREF(retstr);

return -1;

}

static int

multibytecodec_decerror(MultibyteCodec *codec,

MultibyteCodec_State *state,

MultibyteDecodeBuffer *buf,

PyObject *errors, int e)

{

PyObject *argsobj, *retobj = NULL, *retuni = NULL;

int retunisize, newpos;

const char *reason;

size_t esize;

int start, end;

if (e > 0) {

reason = "illegal multibyte sequence";

esize = e;

}

else {

switch (e) {

case MBERR_TOOSMALL:

REQUIRE_DECODEBUFFER(buf, -1);

return 0; /* retry it */

case MBERR_TOOFEW:

reason = "incomplete multibyte sequence";

esize = (size_t)(buf->inbuf_end - buf->inbuf);

break;

case MBERR_INTERNAL:

PyErr_SetString(PyExc_RuntimeError,

"internal codec error");

return -1;

default:

PyErr_SetString(PyExc_RuntimeError,

"unknown runtime error");

return -1;

}

}

if (errors == ERROR_REPLACE) {

REQUIRE_DECODEBUFFER(buf, 1);

*buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;

}

if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {

buf->inbuf += esize;

return 0;

}

start = (int)(buf->inbuf - buf->inbuf_top);

end = start + esize;

/* use cached exception object if available */

if (buf->excobj == NULL) {

buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,

(const char *)buf->inbuf_top,

(int)(buf->inbuf_end - buf->inbuf_top),

start, end, reason);

if (buf->excobj == NULL)

goto errorexit;

}

else

if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||

PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||

PyUnicodeDecodeError_SetReason(buf->excobj, reason))

goto errorexit;

if (errors == ERROR_STRICT) {

PyCodec_StrictErrors(buf->excobj);

goto errorexit;

}

argsobj = PyTuple_New(1);

if (argsobj == NULL)

goto errorexit;

PyTuple_SET_ITEM(argsobj, 0, buf->excobj);

Py_INCREF(buf->excobj);

retobj = PyObject_CallObject(errors, argsobj);

Py_DECREF(argsobj);

if (retobj == NULL)

goto errorexit;

if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||

!PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||

!PyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {

PyErr_SetString(PyExc_ValueError,

"decoding error handler must return "

"(unicode, int) tuple");

goto errorexit;

}

retunisize = PyUnicode_GET_SIZE(retuni);

if (retunisize > 0) {

REQUIRE_DECODEBUFFER(buf, retunisize);

memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),

retunisize * Py_UNICODE_SIZE);

buf->outbuf += retunisize;

}

newpos = (int)PyInt_AS_LONG(PyTuple_GET_ITEM(retobj, 1));

if (newpos < 0)

newpos += (int)(buf->inbuf_end - buf->inbuf_top);

if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {

PyErr_Format(PyExc_IndexError,

"position %d from error handler out of bounds",

newpos);

goto errorexit;

}

buf->inbuf = buf->inbuf_top + newpos;

Py_DECREF(retobj);

return 0;

errorexit:

Py_XDECREF(retobj);

return -1;

}

static PyObject *

multibytecodec_encode(MultibyteCodec *codec,

MultibyteCodec_State *state,

const Py_UNICODE **data, size_t datalen,

PyObject *errors, int flags)

{

MultibyteEncodeBuffer buf;

int finalsize, r = 0;

if (datalen == 0)

return PyString_FromString("");

buf.excobj = NULL;

buf.inbuf = buf.inbuf_top = *data;

buf.inbuf_end = buf.inbuf_top + datalen;

if (datalen > (INT_MAX - 16) / 2) {

PyErr_NoMemory();

goto errorexit;

}

buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);

if (buf.outobj == NULL)

goto errorexit;

buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);

buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);

while (buf.inbuf < buf.inbuf_end) {

size_t inleft, outleft;

/* we don't reuse inleft and outleft here.

* error callbacks can relocate the cursor anywhere on buffer*/

inleft = (size_t)(buf.inbuf_end - buf.inbuf);

outleft = (size_t)(buf.outbuf_end - buf.outbuf);

r = codec->encode(state, codec->config, &buf.inbuf, inleft,

&buf.outbuf, outleft, flags);

*data = buf.inbuf;

if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))

break;

else if (multibytecodec_encerror(codec, state, &buf, errors,r))

goto errorexit;

else if (r == MBERR_TOOFEW)

break;

}

if (codec->encreset != NULL)

for (;;) {

size_t outleft;

outleft = (size_t)(buf.outbuf_end - buf.outbuf);

r = codec->encreset(state, codec->config, &buf.outbuf,

outleft);

if (r == 0)

break;

else if (multibytecodec_encerror(codec, state,

&buf, errors, r))

goto errorexit;

}

finalsize = (int)((char*)buf.outbuf - PyString_AS_STRING(buf.outobj));

if (finalsize != PyString_GET_SIZE(buf.outobj))

if (_PyString_Resize(&buf.outobj, finalsize) == -1)

goto errorexit;

Py_XDECREF(buf.excobj);

return buf.outobj;

errorexit:

Py_XDECREF(buf.excobj);

Py_XDECREF(buf.outobj);

return NULL;

}

static PyObject *

MultibyteCodec_Encode(MultibyteCodecObject *self,

PyObject *args, PyObject *kwargs)

{

MultibyteCodec_State state;

Py_UNICODE *data;

PyObject *errorcb, *r, *arg, *ucvt;

const char *errors = NULL;

int datalen;

if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",

codeckwarglist, &arg, &errors))

return NULL;

if (PyUnicode_Check(arg))

ucvt = NULL;

else {

arg = ucvt = PyObject_Unicode(arg);

if (arg == NULL)

return NULL;

else if (!PyUnicode_Check(arg)) {

PyErr_SetString(PyExc_TypeError,

"couldn't convert the object to unicode.");

Py_DECREF(ucvt);

return NULL;

}

}

data = PyUnicode_AS_UNICODE(arg);

datalen = PyUnicode_GET_SIZE(arg);

errorcb = get_errorcallback(errors);

if (errorcb == NULL) {

Py_XDECREF(ucvt);

return NULL;

}

if (self->codec->encinit != NULL &&

self->codec->encinit(&state, self->codec->config) != 0)

goto errorexit;

r = multibytecodec_encode(self->codec, &state,

(const Py_UNICODE **)&data, datalen, errorcb,

MBENC_FLUSH | MBENC_RESET);

if (r == NULL)

goto errorexit;

if (errorcb > ERROR_MAX) {

Py_DECREF(errorcb);

}

Py_XDECREF(ucvt);

return make_tuple(r, datalen);

errorexit:

if (errorcb > ERROR_MAX) {

Py_DECREF(errorcb);

}

Py_XDECREF(ucvt);

return NULL;

}

static PyObject *

MultibyteCodec_Decode(MultibyteCodecObject *self,

PyObject *args, PyObject *kwargs)

{

MultibyteCodec_State state;

MultibyteDecodeBuffer buf;

PyObject *errorcb;

const char *data, *errors = NULL;

int datalen, finalsize;

if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|z:decode",

codeckwarglist, &data, &datalen, &errors))

return NULL;

errorcb = get_errorcallback(errors);

if (errorcb == NULL)

return NULL;

if (datalen == 0) {

if (errorcb > ERROR_MAX) {

Py_DECREF(errorcb);

}

return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);

}

buf.outobj = buf.excobj = NULL;

buf.inbuf = buf.inbuf_top = (unsigned char *)data;

buf.inbuf_end = buf.inbuf_top + datalen;

buf.outobj = PyUnicode_FromUnicode(NULL, datalen);

if (buf.outobj == NULL)

goto errorexit;

buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);

buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);

if (self->codec->decinit != NULL &&

self->codec->decinit(&state, self->codec->config) != 0)

goto errorexit;

while (buf.inbuf < buf.inbuf_end) {

size_t inleft, outleft;

int r;

inleft = (size_t)(buf.inbuf_end - buf.inbuf);

outleft = (size_t)(buf.outbuf_end - buf.outbuf);

r = self->codec->decode(&state, self->codec->config,

&buf.inbuf, inleft, &buf.outbuf, outleft);

if (r == 0)

break;

else if (multibytecodec_decerror(self->codec, &state,

&buf, errorcb, r))

goto errorexit;

}

finalsize = (int)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));

if (finalsize != PyUnicode_GET_SIZE(buf.outobj))

if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)

goto errorexit;

Py_XDECREF(buf.excobj);

if (errorcb > ERROR_MAX) {

Py_DECREF(errorcb);

}

return make_tuple(buf.outobj, datalen);

errorexit:

if (errorcb > ERROR_MAX) {

Py_DECREF(errorcb);

}

Py_XDECREF(buf.excobj);

Py_XDECREF(buf.outobj);

return NULL;

}

static PyObject *

MultibyteCodec_StreamReader(MultibyteCodecObject *self,

PyObject *args, PyObject *kwargs)

{

PyObject *stream;

char *errors = NULL;

if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s:StreamReader",

streamkwarglist, &stream, &errors))

return NULL;

return mbstreamreader_create(self->codec, stream, errors);

}

static PyObject *

MultibyteCodec_StreamWriter(MultibyteCodecObject *self,

PyObject *args, PyObject *kwargs)

{

PyObject *stream;

char *errors = NULL;

if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|s:StreamWriter",

streamkwarglist, &stream, &errors))

return NULL;

return mbstreamwriter_create(self->codec, stream, errors);

}

static struct PyMethodDef multibytecodec_methods[] = {

{"encode", (PyCFunction)MultibyteCodec_Encode,

METH_VARARGS | METH_KEYWORDS,

MultibyteCodec_Encode__doc__},

{"decode", (PyCFunction)MultibyteCodec_Decode,

METH_VARARGS | METH_KEYWORDS,

MultibyteCodec_Decode__doc__},

{"StreamReader",(PyCFunction)MultibyteCodec_StreamReader,

METH_VARARGS | METH_KEYWORDS,

MultibyteCodec_StreamReader__doc__},

{"StreamWriter",(PyCFunction)MultibyteCodec_StreamWriter,

METH_VARARGS | METH_KEYWORDS,

MultibyteCodec_StreamWriter__doc__},

{NULL, NULL},

};

static void

multibytecodec_dealloc(MultibyteCodecObject *self)

{

PyObject_Del(self);

}

static PyTypeObject MultibyteCodec_Type = {

PyObject_HEAD_INIT(NULL)

0, /* ob_size */

"MultibyteCodec", /* tp_name */

sizeof(MultibyteCodecObject), /* tp_basicsize */

0, /* tp_itemsize */

/* methods */

(destructor)multibytecodec_dealloc, /* tp_dealloc */

0, /* tp_print */

0, /* tp_getattr */

0, /* tp_setattr */

0, /* tp_compare */

0, /* tp_repr */

0, /* tp_as_number */

0, /* tp_as_sequence */

0, /* tp_as_mapping */

0, /* tp_hash */

0, /* tp_call */

0, /* tp_str */

PyObject_GenericGetAttr, /* tp_getattro */

0, /* tp_setattro */

0, /* tp_as_buffer */

Py_TPFLAGS_DEFAULT, /* tp_flags */

0, /* tp_doc */

0, /* tp_traverse */

0, /* tp_clear */

0, /* tp_richcompare */

0, /* tp_weaklistoffset */

0, /* tp_iter */

0, /* tp_iterext */

multibytecodec_methods, /* tp_methods */

};

static PyObject *

mbstreamreader_iread(MultibyteStreamReaderObject *self,

const char *method, int sizehint)

{

MultibyteDecodeBuffer buf;

PyObject *cres;

int rsize, r, finalsize = 0;

if (sizehint == 0)

return PyUnicode_FromUnicode(NULL, 0);

buf.outobj = buf.excobj = NULL;

cres = NULL;

for (;;) {

if (sizehint < 0)

cres = PyObject_CallMethod(self->stream,

(char *)method, NULL);

else

cres = PyObject_CallMethod(self->stream,

(char *)method, "i", sizehint);

if (cres == NULL)

goto errorexit;

if (!PyString_Check(cres)) {

PyErr_SetString(PyExc_TypeError,

"stream function returned a "

"non-string object");

goto errorexit;

}

if (self->pendingsize > 0) {

PyObject *ctr;

char *ctrdata;

if (PyString_GET_SIZE(cres) > INT_MAX - self->pendingsize) {

PyErr_NoMemory();

goto errorexit;

}

rsize = PyString_GET_SIZE(cres) + self->pendingsize;

ctr = PyString_FromStringAndSize(NULL, rsize);

if (ctr == NULL)

goto errorexit;

ctrdata = PyString_AS_STRING(ctr);

memcpy(ctrdata, self->pending, self->pendingsize);

memcpy(ctrdata + self->pendingsize,

PyString_AS_STRING(cres),

PyString_GET_SIZE(cres));

Py_DECREF(cres);

cres = ctr;

self->pendingsize = 0;

}

rsize = PyString_GET_SIZE(cres);

buf.inbuf = buf.inbuf_top =

(unsigned char *)PyString_AS_STRING(cres);

buf.inbuf_end = buf.inbuf_top + rsize;

if (buf.outobj == NULL) {

buf.outobj = PyUnicode_FromUnicode(NULL, rsize);

if (buf.outobj == NULL)

goto errorexit;

buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);

buf.outbuf_end = buf.outbuf +

PyUnicode_GET_SIZE(buf.outobj);

}

r = 0;

if (rsize > 0)

while (buf.inbuf < buf.inbuf_end) {

size_t inleft, outleft;

inleft = (size_t)(buf.inbuf_end - buf.inbuf);

outleft = (size_t)(buf.outbuf_end -buf.outbuf);

r = self->codec->decode(&self->state,

self->codec->config,

&buf.inbuf, inleft,

&buf.outbuf, outleft);

if (r == 0 || r == MBERR_TOOFEW)

break;

else if (multibytecodec_decerror(self->codec,

&self->state, &buf,

self->errors, r))

goto errorexit;

}

if (rsize == 0 || sizehint < 0) { /* end of file */

if (buf.inbuf < buf.inbuf_end &&

multibytecodec_decerror(self->codec, &self->state,

&buf, self->errors, MBERR_TOOFEW))

goto errorexit;

}

if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */

size_t npendings;

/* we can't assume that pendingsize is still 0 here.

* because this function can be called recursively

* from error callback */

npendings = (size_t)(buf.inbuf_end - buf.inbuf);

if (npendings + self->pendingsize > MAXDECPENDING) {

PyErr_SetString(PyExc_RuntimeError,

"pending buffer overflow");

goto errorexit;

}

memcpy(self->pending + self->pendingsize, buf.inbuf,

npendings);

self->pendingsize += npendings;

}

finalsize = (int)(buf.outbuf -

PyUnicode_AS_UNICODE(buf.outobj));

Py_DECREF(cres);

cres = NULL;

if (sizehint < 0 || finalsize != 0 || rsize == 0)

break;

sizehint = 1; /* read 1 more byte and retry */

}

if (finalsize != PyUnicode_GET_SIZE(buf.outobj))

if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)

goto errorexit;

Py_XDECREF(cres);

Py_XDECREF(buf.excobj);

return buf.outobj;

errorexit:

Py_XDECREF(cres);

Py_XDECREF(buf.excobj);

Py_XDECREF(buf.outobj);

return NULL;

}

static PyObject *

mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)

{

PyObject *sizeobj = NULL;

long size;

if (!PyArg_ParseTuple(args, "|O:read", &sizeobj))

return NULL;

if (sizeobj == Py_None || sizeobj == NULL)

size = -1;

else if (PyInt_Check(sizeobj))

size = PyInt_AsLong(sizeobj);

else {

PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");

return NULL;

}

return mbstreamreader_iread(self, "read", size);

}

static PyObject *

mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)

{

PyObject *sizeobj = NULL;

long size;

if (!PyArg_ParseTuple(args, "|O:readline", &sizeobj))

return NULL;

if (sizeobj == Py_None || sizeobj == NULL)

size = -1;

else if (PyInt_Check(sizeobj))

size = PyInt_AsLong(sizeobj);

else {

PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");

return NULL;

}

return mbstreamreader_iread(self, "readline", size);

}

static PyObject *

mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)

{

PyObject *sizehintobj = NULL, *r, *sr;

long sizehint;

if (!PyArg_ParseTuple(args, "|O:readlines", &sizehintobj))

return NULL;

if (sizehintobj == Py_None || sizehintobj == NULL)

sizehint = -1;

else if (PyInt_Check(sizehintobj))

sizehint = PyInt_AsLong(sizehintobj);

else {

PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");

return NULL;

}

r = mbstreamreader_iread(self, "read", sizehint);

if (r == NULL)

return NULL;

sr = PyUnicode_Splitlines(r, 1);

Py_DECREF(r);

return sr;

}

static PyObject *

mbstreamreader_reset(MultibyteStreamReaderObject *self)

{

if (self->codec->decreset != NULL &&

self->codec->decreset(&self->state, self->codec->config) != 0)

return NULL;

self->pendingsize = 0;

Py_INCREF(Py_None);

return Py_None;

}

static struct PyMethodDef mbstreamreader_methods[] = {

{"read", (PyCFunction)mbstreamreader_read,

METH_VARARGS, NULL},

{"readline", (PyCFunction)mbstreamreader_readline,

METH_VARARGS, NULL},

{"readlines", (PyCFunction)mbstreamreader_readlines,

METH_VARARGS, NULL},

{"reset", (PyCFunction)mbstreamreader_reset,

METH_NOARGS, NULL},

{NULL, NULL},

};

static void

mbstreamreader_dealloc(MultibyteStreamReaderObject *self)

{

if (self->errors > ERROR_MAX) {

Py_DECREF(self->errors);

}

Py_DECREF(self->stream);

PyObject_Del(self);

}

static PyTypeObject MultibyteStreamReader_Type = {

PyObject_HEAD_INIT(NULL)

0, /* ob_size */

"MultibyteStreamReader", /* tp_name */

sizeof(MultibyteStreamReaderObject), /* tp_basicsize */

0, /* tp_itemsize */

/* methods */

(destructor)mbstreamreader_dealloc, /* tp_dealloc */

0, /* tp_print */

0, /* tp_getattr */

0, /* tp_setattr */

0, /* tp_compare */

0, /* tp_repr */

0, /* tp_as_number */

0, /* tp_as_sequence */

0, /* tp_as_mapping */

0, /* tp_hash */

0, /* tp_call */

0, /* tp_str */

PyObject_GenericGetAttr, /* tp_getattro */

0, /* tp_setattro */

0, /* tp_as_buffer */

Py_TPFLAGS_DEFAULT, /* tp_flags */

0, /* tp_doc */

0, /* tp_traverse */

0, /* tp_clear */

0, /* tp_richcompare */

0, /* tp_weaklistoffset */

0, /* tp_iter */

0, /* tp_iterext */

mbstreamreader_methods, /* tp_methods */

};

static int

mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,

PyObject *unistr)

{

PyObject *wr, *ucvt, *r = NULL;

Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;

int datalen;

if (PyUnicode_Check(unistr))

ucvt = NULL;

else {

unistr = ucvt = PyObject_Unicode(unistr);

if (unistr == NULL)

return -1;

else if (!PyUnicode_Check(unistr)) {

PyErr_SetString(PyExc_TypeError,

"couldn't convert the object to unicode.");

Py_DECREF(ucvt);

return -1;

}

}

datalen = PyUnicode_GET_SIZE(unistr);

if (datalen == 0) {