Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Remove CJKCodecs implementation of UTF-7 and UTF-8 codec which 

aren't intended to be part of Python distributiuon.  This was
accidently imported on mass converting from standalone version of
CJKCodecs.
This commit is contained in:
Hye-Shik Chang 2004-07-28 09:36:52 +00:00
parent bee417439e
commit eb34110190
1 changed files with 0 additions and 560 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

560
Modules/cjkcodecs/_codecs_unicode.c
View File

@ -1,560 +0,0 @@
/*
* _codecs_unicode.c: Codecs collection for Unicode encodings
*
* Written by Hye-Shik Chang <perky@FreeBSD.org>
* $CJKCodecs: _codecs_unicode.c,v 1.5 2004/06/27 21:41:15 perky Exp $
*/
#include "cjkcodecs.h"
/*
* UTF-7 codec
*/
#define SET_DIRECT 1
#define SET_OPTIONAL 2
#define SET_WHITESPACE 3
#define _D SET_DIRECT
#define _O SET_OPTIONAL
#define _W SET_WHITESPACE
static const char utf7_sets[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, _W, _W, 0, 0, _W, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
_W, _O, _O, _O, _O, _O, _O, _D, _D, _D, _O, 0, _D, _D, _D, 0,
_D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, _O, _O, _O, _D,
_O, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D,
_D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, 0, _O, _O, _O,
_O, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D,
_D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _D, _O, _O, _O, 0, 0,
};
#undef _W
#undef _O
#undef _D
#define B64(n) ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" \
"0123456789+/"[(n) & 0x3f])
#define B64CHAR(c) (((c) >= 'A' && (c) <= 'Z') || \
((c) >= 'a' && (c) <= 'z') || \
((c) >= '0' && (c) <= '9') || \
(c) == '+' || (c) == '/')
#define UB64(c) ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
(c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4)
#define UTF7_DENCODABLE_COMPATIBLE(c) (utf7_sets[c] != 0)
#define UTF7_DENCODABLE_STRICT(c) (utf7_sets[c] == SET_DIRECT || \
utf7_sets[c] == SET_WHITESPACE)
#define ESTATE_INITIALIZE(state) \
ESTATE_SETSTAGE(state, 0) \
ESTATE_CLEARSHIFTED(state)
#define ESTATE_SETPENDING(state, v) (state)->c[0] = (v);
#define ESTATE_GETPENDING(state) (state)->c[0]
#define ESTATE_SETSHIFTED(state) (state)->c[2] = 1;
#define ESTATE_ISSHIFTED(state) ((state)->c[2])
#define ESTATE_CLEARSHIFTED(state) (state)->c[2] = 0;
#define ESTATE_SETSTAGE(state, v) (state)->c[3] = (v);
#define ESTATE_GETSTAGE(state) ((state)->c[3])
ENCODER_INIT(utf_7)
{
ESTATE_INITIALIZE(state)
return 0;
}
ENCODER_RESET(utf_7)
{
if (ESTATE_ISSHIFTED(state)) {
if (ESTATE_GETSTAGE(state) != 0) {
unsigned char oc;
oc = B64(ESTATE_GETPENDING(state));
WRITE2(oc, '-')
NEXT_OUT(2)
}
else {
WRITE1('-')
NEXT_OUT(1)
}
ESTATE_CLEARSHIFTED(state)
}
return 0;
}
ENCODER(utf_7)
{
while (inleft > 0) {
Py_UNICODE c1 = IN1, c2 = 0;
size_t insize = 1;
#if Py_UNICODE_SIZE == 2
if (c1 >> 10 == 0xd800 >> 10) { /* high surrogate */
REQUIRE_INBUF(2)
if (IN2 >> 10 != 0xdc00 >> 10) /* low surrogate */
return 2; /* invalid surrogate pair */
c2 = IN2;
insize = 2;
}
#else
if (c1 > 0x10ffff) /* UTF-16 unencodable */
return 1;
else if (c1 > 0xffff) {
c2 = 0xdc00 | ((c1 - 0x10000) & 0x3ff);
c1 = 0xd800 | ((c1 - 0x10000) >> 10);
}
#endif
for (;;) {
unsigned char oc1, oc2, oc3;
if (ESTATE_ISSHIFTED(state)) {
if (c1 < 128 && UTF7_DENCODABLE_STRICT(c1)) {
if (ESTATE_GETSTAGE(state) != 0) {
oc1 = B64(ESTATE_GETPENDING(
state));
WRITE3(oc1, '-',
(unsigned char)c1)
NEXT_OUT(3)
} else {
WRITE2('-',
(unsigned char)c1)
NEXT_OUT(2)
}
ESTATE_CLEARSHIFTED(state)
} else {
switch (ESTATE_GETSTAGE(state)) {
case 0:
oc1 = c1 >> 10;
oc2 = (c1 >> 4) & 0x3f;
WRITE2(B64(oc1), B64(oc2))
ESTATE_SETPENDING(state,
(c1 & 0x0f) << 2)
ESTATE_SETSTAGE(state, 2)
NEXT_OUT(2)
break;
case 1:
oc1 = ESTATE_GETPENDING(state)
| (c1 >> 12);
oc2 = (c1 >> 6) & 0x3f;
oc3 = c1 & 0x3f;
WRITE3(B64(oc1), B64(oc2),
B64(oc3))
ESTATE_SETSTAGE(state, 0)
NEXT_OUT(3)
break;
case 2:
oc1 = ESTATE_GETPENDING(state)
| (c1 >> 14);
oc2 = (c1 >> 8) & 0x3f;
oc3 = (c1 >> 2) & 0x3f;
WRITE3(B64(oc1), B64(oc2),
B64(oc3))
ESTATE_SETPENDING(state,
(c1 & 0x03) << 4)
ESTATE_SETSTAGE(state, 1)
NEXT_OUT(3)
break;
default:
return MBERR_INTERNAL;
}
}
}
else {
if (c1 < 128 && UTF7_DENCODABLE_STRICT(c1)) {
WRITE1((unsigned char)c1)
NEXT_OUT(1)
}
else if (c1 == '+') {
WRITE2('+', '-')
NEXT_OUT(2)
}
else {
oc1 = c1 >> 10;
oc2 = (c1 >> 4) & 0x3f;
WRITE3('+', B64(oc1), B64(oc2))
ESTATE_SETPENDING(state,
(c1 & 0x0f) << 2)
ESTATE_SETSTAGE(state, 2)
ESTATE_SETSHIFTED(state)
NEXT_OUT(3)
}
}
if (c2 != 0) {
c1 = c2;
c2 = 0;
}
else
break;
}
NEXT_IN(insize)
}
return 0;
}
#define DSTATE_INITIALIZE(state) \
DSTATE_SETBSTAGE(state, 0) \
DSTATE_CLEARSHIFTED(state) \
DSTATE_SETULENGTH(state, 0) \
DSTATE_SETUPENDING1(state, 0) \
DSTATE_SETUPENDING2(state, 0)
/* XXX: Type-mixed usage of a state union may be not so portable.
* If you see any problem with this on your platfom. Please let
* me know. */
#define DSTATE_SETSHIFTED(state) (state)->c[0] = 1;
#define DSTATE_ISSHIFTED(state) ((state)->c[0])
#define DSTATE_CLEARSHIFTED(state) (state)->c[0] = 0;
#define DSTATE_SETBSTAGE(state, v) (state)->c[1] = (v);
#define DSTATE_GETBSTAGE(state) ((state)->c[1])
#define DSTATE_SETBPENDING(state, v) (state)->c[2] = (v);
#define DSTATE_GETBPENDING(state) ((state)->c[2])
#define DSTATE_SETULENGTH(state, v) (state)->c[3] = (v);
#define DSTATE_GETULENGTH(state) ((state)->c[3])
#define DSTATE_SETUPENDING1(state, v) (state)->u2[2] = (v);
#define DSTATE_GETUPENDING1(state) (state)->u2[2]
#define DSTATE_SETUPENDING2(state, v) (state)->u2[3] = (v);
#define DSTATE_GETUPENDING2(state) (state)->u2[3]
#define DSTATE_UAPPEND(state, v) \
(state)->u2[(state)->c[3] > 1 ? 3 : 2] |= \
((state)->c[3] & 1) ? (v) : ((ucs2_t)(v)) << 8; \
(state)->c[3]++;
DECODER_INIT(utf_7)
{
DSTATE_INITIALIZE(state)
return 0;
}
static int
utf_7_flush(MultibyteCodec_State *state,
Py_UNICODE **outbuf, size_t *outleft)
{
switch (DSTATE_GETULENGTH(state)) {
case 2: {
ucs2_t uc;
uc = DSTATE_GETUPENDING1(state);
#if Py_UNICODE_SIZE == 4
if (uc >> 10 == 0xd800 >> 10)
return MBERR_TOOFEW;
#endif
OUT1(uc)
(*outbuf)++;
(*outleft)--;
DSTATE_SETULENGTH(state, 0)
DSTATE_SETUPENDING1(state, 0)
break;
}
#if Py_UNICODE_SIZE == 4
case 4:
if (DSTATE_GETUPENDING2(state) >> 10 != 0xdc00 >> 10)
return 1;
OUT1(0x10000 + (((ucs4_t)DSTATE_GETUPENDING1(state) - 0xd800)
<< 10) + (DSTATE_GETUPENDING2(state) - 0xdc00))
(*outbuf)++;
(*outleft)--;
DSTATE_SETULENGTH(state, 0)
DSTATE_SETUPENDING1(state, 0)
DSTATE_SETUPENDING2(state, 0)
break;
#endif
case 0: /* FALLTHROUGH */
case 1: /* FALLTHROUGH */
case 3:
return MBERR_TOOFEW;
default:
return MBERR_INTERNAL;
}
return 0;
}
DECODER_RESET(utf_7)
{
DSTATE_INITIALIZE(state)
return 0;
}
DECODER(utf_7)
{
while (inleft > 0) {
unsigned char c = IN1;
int r;
if (!DSTATE_ISSHIFTED(state)) {
if (c == '+') {
REQUIRE_INBUF(2)
if (inleft >= 2 && IN2 == '-') {
WRITE1('+')
NEXT(2, 1)
}
else {
DSTATE_SETSHIFTED(state)
NEXT_IN(1)
}
}
else if (c < 128 && UTF7_DENCODABLE_COMPATIBLE(c)) {
WRITE1(c)
NEXT(1, 1)
}
else
return 1;
}
else if (B64CHAR(c)) {
unsigned char tb;
REQUIRE_OUTBUF(1)
c = UB64(c);
assert(DSTATE_GETULENGTH(state) < 4);
switch (DSTATE_GETBSTAGE(state)) {
case 0:
DSTATE_SETBPENDING(state, c << 2)
DSTATE_SETBSTAGE(state, 1)
break;
case 1:
tb = DSTATE_GETBPENDING(state) | (c >> 4);
DSTATE_SETBPENDING(state, c << 4)
DSTATE_SETBSTAGE(state, 2)
DSTATE_UAPPEND(state, tb)
break;
case 2:
tb = DSTATE_GETBPENDING(state) | (c >> 2);
DSTATE_SETBPENDING(state, c << 6)
DSTATE_SETBSTAGE(state, 3)
DSTATE_UAPPEND(state, tb)
break;
case 3:
tb = DSTATE_GETBPENDING(state) | c;
DSTATE_SETBSTAGE(state, 0)
DSTATE_UAPPEND(state, tb)
break;
}
r = utf_7_flush(state, outbuf, &outleft);
if (r != 0 && r != MBERR_TOOFEW)
return r;
NEXT_IN(1)
}
else if (c == '-' || UTF7_DENCODABLE_COMPATIBLE(c)) {
if (DSTATE_GETBSTAGE(state) != 0) {
DSTATE_UAPPEND(state, DSTATE_GETBSTAGE(state))
DSTATE_SETBSTAGE(state, 0)
}
r = utf_7_flush(state, outbuf, &outleft);
if (r != 0 && r != MBERR_TOOFEW)
return r;
DSTATE_CLEARSHIFTED(state)
if (c != '-') {
WRITE1(c)
NEXT_OUT(1)
}
NEXT_IN(1)
}
else
return 1;
}
return 0;
}
/*
* UTF-8 codec
*/
ENCODER(utf_8)
{
while (inleft > 0) {
ucs4_t c = **inbuf;
size_t outsize, insize = 1;
if (c < 0x80) outsize = 1;
else if (c < 0x800) outsize = 2;
else {
#if Py_UNICODE_SIZE == 2
if (c >> 10 == 0xd800 >> 10) { /* high surrogate */
if (inleft < 2) {
if (!(flags & MBENC_FLUSH))
return MBERR_TOOFEW;
}
else if ((*inbuf)[1] >> 10 == 0xdc00 >> 10) {
/* low surrogate */
c = 0x10000 + ((c - 0xd800) << 10) +
((ucs4_t)((*inbuf)[1]) - 0xdc00);
insize = 2;
}
}
#endif
if (c < 0x10000) outsize = 3;
else if (c < 0x200000) outsize = 4;
else if (c < 0x4000000) outsize = 5;
else outsize = 6;
}
REQUIRE_OUTBUF(outsize)
switch (outsize) {
case 6:
(*outbuf)[5] = 0x80 | (c & 0x3f);
c = c >> 6;
c |= 0x4000000;
/* FALLTHROUGH */
case 5:
(*outbuf)[4] = 0x80 | (c & 0x3f);
c = c >> 6;
c |= 0x200000;
/* FALLTHROUGH */
case 4:
(*outbuf)[3] = 0x80 | (c & 0x3f);
c = c >> 6;
c |= 0x10000;
/* FALLTHROUGH */
case 3:
(*outbuf)[2] = 0x80 | (c & 0x3f);
c = c >> 6;
c |= 0x800;
/* FALLTHROUGH */
case 2:
(*outbuf)[1] = 0x80 | (c & 0x3f);
c = c >> 6;
c |= 0xc0;
/* FALLTHROUGH */
case 1:
(*outbuf)[0] = c;
}
NEXT(insize, outsize)
}
return 0;
}
DECODER(utf_8)
{
while (inleft > 0) {
unsigned char c = **inbuf;
REQUIRE_OUTBUF(1)
if (c < 0x80) {
(*outbuf)[0] = (unsigned char)c;
NEXT(1, 1)
}
else if (c < 0xc2) {
return 1;
}
else if (c < 0xe0) {
unsigned char c2;
REQUIRE_INBUF(2)
c2 = (*inbuf)[1];
if (!((c2 ^ 0x80) < 0x40))
return 2;
**outbuf = ((Py_UNICODE)(c & 0x1f) << 6) |
(Py_UNICODE)(c2 ^ 0x80);
NEXT(2, 1)
}
else if (c < 0xf0) {
unsigned char c2, c3;
REQUIRE_INBUF(3)
c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
if (!((c2 ^ 0x80) < 0x40 &&
(c3 ^ 0x80) < 0x40 && (c >= 0xe1 || c2 >= 0xa0)))
return 3;
**outbuf = ((Py_UNICODE)(c & 0x0f) << 12)
| ((Py_UNICODE)(c2 ^ 0x80) << 6)
| (Py_UNICODE)(c3 ^ 0x80);
NEXT(3, 1)
}
else if (c < 0xf8) {
unsigned char c2, c3, c4;
ucs4_t code;
REQUIRE_INBUF(4)
c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
c4 = (*inbuf)[3];
if (!((c2 ^ 0x80) < 0x40 &&
(c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
(c >= 0xf1 || c2 >= 0x90)))
return 4;
code = ((ucs4_t)(c & 0x07) << 18)
| ((ucs4_t)(c2 ^ 0x80) << 12)
| ((ucs4_t)(c3 ^ 0x80) << 6)
| (ucs4_t)(c4 ^ 0x80);
WRITEUCS4(code)
NEXT_IN(4)
}
else if (c < 0xfc) {
unsigned char c2, c3, c4, c5;
ucs4_t code;
REQUIRE_INBUF(5)
c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
c4 = (*inbuf)[3]; c5 = (*inbuf)[4];
if (!((c2 ^ 0x80) < 0x40 &&
(c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
(c5 ^ 0x80) < 0x40 && (c >= 0xf9 || c2 >= 0x88)))
return 5;
code = ((ucs4_t)(c & 0x03) << 24)
| ((ucs4_t)(c2 ^ 0x80) << 18)
| ((ucs4_t)(c3 ^ 0x80) << 12)
| ((ucs4_t)(c4 ^ 0x80) << 6)
| (ucs4_t)(c5 ^ 0x80);
WRITEUCS4(code)
NEXT_IN(5)
}
else if (c < 0xff) {
unsigned char c2, c3, c4, c5, c6;
ucs4_t code;
REQUIRE_INBUF(6)
c2 = (*inbuf)[1]; c3 = (*inbuf)[2];
c4 = (*inbuf)[3]; c5 = (*inbuf)[4];
c6 = (*inbuf)[5];
if (!((c2 ^ 0x80) < 0x40 &&
(c3 ^ 0x80) < 0x40 && (c4 ^ 0x80) < 0x40 &&
(c5 ^ 0x80) < 0x40 && (c6 ^ 0x80) < 0x40 &&
(c >= 0xfd || c2 >= 0x84)))
return 6;
code = ((ucs4_t)(c & 0x01) << 30)
| ((ucs4_t)(c2 ^ 0x80) << 24)
| ((ucs4_t)(c3 ^ 0x80) << 18)
| ((ucs4_t)(c4 ^ 0x80) << 12)
| ((ucs4_t)(c5 ^ 0x80) << 6)
| (ucs4_t)(c6 ^ 0x80);
WRITEUCS4(code)
NEXT_IN(6)
}
else
return 1;
}
return 0;
}
BEGIN_MAPPINGS_LIST
END_MAPPINGS_LIST
BEGIN_CODECS_LIST
CODEC_STATEFUL(utf_7)
CODEC_STATELESS(utf_8)
END_CODECS_LIST
I_AM_A_MODULE_FOR(unicode)