cpython/Modules/cjkcodecs/_codecs_iso2022.c

/*
 * _codecs_iso2022.c: Codecs collection for ISO-2022 encodings.
 *
 * Written by Hye-Shik Chang <perky@FreeBSD.org>
 */

#define USING_IMPORTED_MAPS
#define USING_BINARY_PAIR_SEARCH
#define EXTERN_JISX0213_PAIR
#define EMULATE_JISX0213_2000_ENCODE_INVALID MAP_UNMAPPABLE
#define EMULATE_JISX0213_2000_DECODE_INVALID MAP_UNMAPPABLE

#include "cjkcodecs.h"
#include "alg_jisx0201.h"
#include "emu_jisx0213_2000.h"
#include "mappings_jisx0213_pair.h"

/* STATE

   state->c[0-3]

	00000000
	||^^^^^|
	|+-----+----  G0-3 Character Set
	+-----------  Is G0-3 double byte?

   state->c[4]

	00000000
	      ||
	      |+----  Locked-Shift?
	      +-----  ESC Throughout
*/

#define ESC			0x1B
#define SO			0x0E
#define SI			0x0F
#define LF			0x0A

#define MAX_ESCSEQLEN		16

#define CHARSET_ISO8859_1	'A'
#define CHARSET_ASCII		'B'
#define CHARSET_ISO8859_7	'F'
#define CHARSET_JISX0201_K	'I'
#define CHARSET_JISX0201_R	'J'

#define CHARSET_GB2312		('A'|CHARSET_DBCS)
#define CHARSET_JISX0208	('B'|CHARSET_DBCS)
#define CHARSET_KSX1001		('C'|CHARSET_DBCS)
#define CHARSET_JISX0212	('D'|CHARSET_DBCS)
#define CHARSET_GB2312_8565	('E'|CHARSET_DBCS)
#define CHARSET_CNS11643_1	('G'|CHARSET_DBCS)
#define CHARSET_CNS11643_2	('H'|CHARSET_DBCS)
#define CHARSET_JISX0213_2000_1	('O'|CHARSET_DBCS)
#define CHARSET_JISX0213_2	('P'|CHARSET_DBCS)
#define CHARSET_JISX0213_2004_1	('Q'|CHARSET_DBCS)
#define CHARSET_JISX0208_O	('@'|CHARSET_DBCS)

#define CHARSET_DBCS		0x80
#define ESCMARK(mark)		((mark) & 0x7f)

#define IS_ESCEND(c)	(((c) >= 'A' && (c) <= 'Z') || (c) == '@')
#define IS_ISO2022ESC(c2) \
		((c2) == '(' || (c2) == ')' || (c2) == '$' || \
		 (c2) == '.' || (c2) == '&')
	/* this is not a complete list of ISO-2022 escape sequence headers.
	 * but, it's enough to implement CJK instances of iso-2022. */

#define MAP_UNMAPPABLE		0xFFFF
#define MAP_MULTIPLE_AVAIL	0xFFFE /* for JIS X 0213 */

#define F_SHIFTED		0x01
#define F_ESCTHROUGHOUT		0x02

#define STATE_SETG(dn, v)	((state)->c[dn]) = (v);
#define STATE_GETG(dn)		((state)->c[dn])

#define STATE_G0		STATE_GETG(0)
#define STATE_G1		STATE_GETG(1)
#define STATE_G2		STATE_GETG(2)
#define STATE_G3		STATE_GETG(3)
#define STATE_SETG0(v)		STATE_SETG(0, v)
#define STATE_SETG1(v)		STATE_SETG(1, v)
#define STATE_SETG2(v)		STATE_SETG(2, v)
#define STATE_SETG3(v)		STATE_SETG(3, v)

#define STATE_SETFLAG(f)	((state)->c[4]) |= (f);
#define STATE_GETFLAG(f)	((state)->c[4] & (f))
#define STATE_CLEARFLAG(f)	((state)->c[4]) &= ~(f);
#define STATE_CLEARFLAGS()	((state)->c[4]) = 0;

#define ISO2022_CONFIG		((const struct iso2022_config *)config)
#define CONFIG_ISSET(flag)	(ISO2022_CONFIG->flags & (flag))
#define CONFIG_DESIGNATIONS	(ISO2022_CONFIG->designations)

/* iso2022_config.flags */
#define NO_SHIFT		0x01
#define USE_G2			0x02
#define USE_JISX0208_EXT	0x04

/*-*- internal data structures -*-*/

typedef int (*iso2022_init_func)(void);
typedef ucs4_t (*iso2022_decode_func)(const unsigned char *data);
typedef DBCHAR (*iso2022_encode_func)(const ucs4_t *data, Py_ssize_t *length);

struct iso2022_designation {
	unsigned char mark;
	unsigned char plane;
	unsigned char width;
	iso2022_init_func initializer;
	iso2022_decode_func decoder;
	iso2022_encode_func encoder;
};

struct iso2022_config {
	int flags;
	const struct iso2022_designation *designations; /* non-ascii desigs */
};

/*-*- iso-2022 codec implementation -*-*/

CODEC_INIT(iso2022)
{
	const struct iso2022_designation *desig = CONFIG_DESIGNATIONS;
	for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++)
		if (desig->initializer != NULL && desig->initializer() != 0)
			return -1;
	return 0;
}

ENCODER_INIT(iso2022)
{
	STATE_CLEARFLAGS()
	STATE_SETG0(CHARSET_ASCII)
	STATE_SETG1(CHARSET_ASCII)
	return 0;
}

ENCODER_RESET(iso2022)
{
	if (STATE_GETFLAG(F_SHIFTED)) {
		WRITE1(SI)
		NEXT_OUT(1)
		STATE_CLEARFLAG(F_SHIFTED)
	}
	if (STATE_G0 != CHARSET_ASCII) {
		WRITE3(ESC, '(', 'B')
		NEXT_OUT(3)
		STATE_SETG0(CHARSET_ASCII)
	}
	return 0;
}

ENCODER(iso2022)
{
	while (inleft > 0) {
		const struct iso2022_designation *dsg;
		DBCHAR encoded;
		ucs4_t c = **inbuf;
		Py_ssize_t insize;

		if (c < 0x80) {
			if (STATE_G0 != CHARSET_ASCII) {
				WRITE3(ESC, '(', 'B')
				STATE_SETG0(CHARSET_ASCII)
				NEXT_OUT(3)
			}
			if (STATE_GETFLAG(F_SHIFTED)) {
				WRITE1(SI)
				STATE_CLEARFLAG(F_SHIFTED)
				NEXT_OUT(1)
			}
			WRITE1((unsigned char)c)
			NEXT(1, 1)
			continue;
		}

		DECODE_SURROGATE(c)
		insize = GET_INSIZE(c);

		encoded = MAP_UNMAPPABLE;
		for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
			Py_ssize_t length = 1;
			encoded = dsg->encoder(&c, &length);
			if (encoded == MAP_MULTIPLE_AVAIL) {
				/* this implementation won't work for pair
				 * of non-bmp characters. */
				if (inleft < 2) {
					if (!(flags & MBENC_FLUSH))
						return MBERR_TOOFEW;
					length = -1;
				}
				else
					length = 2;
#if Py_UNICODE_SIZE == 2
				if (length == 2) {
					ucs4_t u4in[2];
					u4in[0] = (ucs4_t)IN1;
					u4in[1] = (ucs4_t)IN2;
					encoded = dsg->encoder(u4in, &length);
				} else
					encoded = dsg->encoder(&c, &length);
#else
				encoded = dsg->encoder(*inbuf, &length);
#endif
				if (encoded != MAP_UNMAPPABLE) {
					insize = length;
					break;
				}
			}
			else if (encoded != MAP_UNMAPPABLE)
				break;
		}

		if (!dsg->mark)
			return 1;
		assert(dsg->width == 1 || dsg->width == 2);

		switch (dsg->plane) {
		case 0: /* G0 */
			if (STATE_GETFLAG(F_SHIFTED)) {
				WRITE1(SI)
				STATE_CLEARFLAG(F_SHIFTED)
				NEXT_OUT(1)
			}
			if (STATE_G0 != dsg->mark) {
				if (dsg->width == 1) {
					WRITE3(ESC, '(', ESCMARK(dsg->mark))
					STATE_SETG0(dsg->mark)
					NEXT_OUT(3)
				}
				else if (dsg->mark == CHARSET_JISX0208) {
					WRITE3(ESC, '$', ESCMARK(dsg->mark))
					STATE_SETG0(dsg->mark)
					NEXT_OUT(3)
				}
				else {
					WRITE4(ESC, '$', '(',
						ESCMARK(dsg->mark))
					STATE_SETG0(dsg->mark)
					NEXT_OUT(4)
				}
			}
			break;
		case 1: /* G1 */
			if (STATE_G1 != dsg->mark) {
				if (dsg->width == 1) {
					WRITE3(ESC, ')', ESCMARK(dsg->mark))
					STATE_SETG1(dsg->mark)
					NEXT_OUT(3)
				}
				else {
					WRITE4(ESC, '$', ')',
						ESCMARK(dsg->mark))
					STATE_SETG1(dsg->mark)
					NEXT_OUT(4)
				}
			}
			if (!STATE_GETFLAG(F_SHIFTED)) {
				WRITE1(SO)
				STATE_SETFLAG(F_SHIFTED)
				NEXT_OUT(1)
			}
			break;
		default: /* G2 and G3 is not supported: no encoding in
			  * CJKCodecs are using them yet */
			return MBERR_INTERNAL;
		}

		if (dsg->width == 1) {
			WRITE1((unsigned char)encoded)
			NEXT_OUT(1)
		}
		else {
			WRITE2(encoded >> 8, encoded & 0xff)
			NEXT_OUT(2)
		}
		NEXT_IN(insize)
	}

	return 0;
}

DECODER_INIT(iso2022)
{
	STATE_CLEARFLAGS()
	STATE_SETG0(CHARSET_ASCII)
	STATE_SETG1(CHARSET_ASCII)
	STATE_SETG2(CHARSET_ASCII)
	return 0;
}

DECODER_RESET(iso2022)
{
	STATE_SETG0(CHARSET_ASCII)
	STATE_CLEARFLAG(F_SHIFTED)
	return 0;
}

static Py_ssize_t
iso2022processesc(const void *config, MultibyteCodec_State *state,
		  const unsigned char **inbuf, Py_ssize_t *inleft)
{
	unsigned char charset, designation;
	Py_ssize_t i, esclen;

	for (i = 1;i < MAX_ESCSEQLEN;i++) {
		if (i >= *inleft)
			return MBERR_TOOFEW;
		if (IS_ESCEND((*inbuf)[i])) {
			esclen = i + 1;
			break;
		}
		else if (CONFIG_ISSET(USE_JISX0208_EXT) && i+1 < *inleft &&
			 (*inbuf)[i] == '&' && (*inbuf)[i+1] == '@')
			i += 2;
	}

	if (i >= MAX_ESCSEQLEN)
		return 1; /* unterminated escape sequence */

	switch (esclen) {
	case 3:
		if (IN2 == '$') {
			charset = IN3 | CHARSET_DBCS;
			designation = 0;
		}
		else {
			charset = IN3;
			if (IN2 == '(') designation = 0;
			else if (IN2 == ')') designation = 1;
			else if (CONFIG_ISSET(USE_G2) && IN2 == '.')
				designation = 2;
			else return 3;
		}
		break;
	case 4:
		if (IN2 != '$')
			return 4;

		charset = IN4 | CHARSET_DBCS;
		if (IN3 == '(') designation = 0;
		else if (IN3 == ')') designation = 1;
		else return 4;
		break;
	case 6: /* designation with prefix */
		if (CONFIG_ISSET(USE_JISX0208_EXT) &&
		    (*inbuf)[3] == ESC && (*inbuf)[4] == '$' &&
		    (*inbuf)[5] == 'B') {
			charset = 'B' | CHARSET_DBCS;
			designation = 0;
		}
		else
			return 6;
		break;
	default:
		return esclen;
	}

	/* raise error when the charset is not designated for this encoding */
	if (charset != CHARSET_ASCII) {
		const struct iso2022_designation *dsg;

		for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++)
			if (dsg->mark == charset)
				break;
		if (!dsg->mark)
			return esclen;
	}

	STATE_SETG(designation, charset)
	*inleft -= esclen;
	(*inbuf) += esclen;
	return 0;
}

#define ISO8859_7_DECODE(c, assi)					\
	if ((c) < 0xa0) (assi) = (c);					\
	else if ((c) < 0xc0 && (0x288f3bc9L & (1L << ((c)-0xa0))))	\
		(assi) = (c);						\
	else if ((c) >= 0xb4 && (c) <= 0xfe && ((c) >= 0xd4 ||		\
		 (0xbffffd77L & (1L << ((c)-0xb4)))))			\
		(assi) = 0x02d0 + (c);					\
	else if ((c) == 0xa1) (assi) = 0x2018;				\
	else if ((c) == 0xa2) (assi) = 0x2019;				\
	else if ((c) == 0xaf) (assi) = 0x2015;

static Py_ssize_t
iso2022processg2(const void *config, MultibyteCodec_State *state,
		 const unsigned char **inbuf, Py_ssize_t *inleft,
		 Py_UNICODE **outbuf, Py_ssize_t *outleft)
{
	/* not written to use encoder, decoder functions because only few
	 * encodings use G2 designations in CJKCodecs */
	if (STATE_G2 == CHARSET_ISO8859_1) {
		if (IN3 < 0x80)
			OUT1(IN3 + 0x80)
		else
			return 3;
	}
	else if (STATE_G2 == CHARSET_ISO8859_7) {
		ISO8859_7_DECODE(IN3 ^ 0x80, **outbuf)
		else return 3;
	}
	else if (STATE_G2 == CHARSET_ASCII) {
		if (IN3 & 0x80) return 3;
		else **outbuf = IN3;
	}
	else
		return MBERR_INTERNAL;

	(*inbuf) += 3;
	*inleft -= 3;
	(*outbuf) += 1;
	*outleft -= 1;
	return 0;
}

DECODER(iso2022)
{
	const struct iso2022_designation *dsgcache = NULL;

	while (inleft > 0) {
		unsigned char c = IN1;
		Py_ssize_t err;

		if (STATE_GETFLAG(F_ESCTHROUGHOUT)) {
			/* ESC throughout mode:
			 * for non-iso2022 escape sequences */
			WRITE1(c) /* assume as ISO-8859-1 */
			NEXT(1, 1)
			if (IS_ESCEND(c)) {
				STATE_CLEARFLAG(F_ESCTHROUGHOUT)
			}
			continue;
		}

		switch (c) {
		case ESC:
			REQUIRE_INBUF(2)
			if (IS_ISO2022ESC(IN2)) {
				err = iso2022processesc(config, state,
							inbuf, &inleft);
				if (err != 0)
					return err;
			}
			else if (CONFIG_ISSET(USE_G2) && IN2 == 'N') {/* SS2 */
				REQUIRE_INBUF(3)
				err = iso2022processg2(config, state,
					inbuf, &inleft, outbuf, &outleft);
				if (err != 0)
					return err;
			}
			else {
				WRITE1(ESC)
				STATE_SETFLAG(F_ESCTHROUGHOUT)
				NEXT(1, 1)
			}
			break;
		case SI:
			if (CONFIG_ISSET(NO_SHIFT))
				goto bypass;
			STATE_CLEARFLAG(F_SHIFTED)
			NEXT_IN(1)
			break;
		case SO:
			if (CONFIG_ISSET(NO_SHIFT))
				goto bypass;
			STATE_SETFLAG(F_SHIFTED)
			NEXT_IN(1)
			break;
		case LF:
			STATE_CLEARFLAG(F_SHIFTED)
			WRITE1(LF)
			NEXT(1, 1)
			break;
		default:
			if (c < 0x20) /* C0 */
				goto bypass;
			else if (c >= 0x80)
				return 1;
			else {
				const struct iso2022_designation *dsg;
				unsigned char charset;
				ucs4_t decoded;

				if (STATE_GETFLAG(F_SHIFTED))
					charset = STATE_G1;
				else
					charset = STATE_G0;

				if (charset == CHARSET_ASCII) {
bypass:					WRITE1(c)
					NEXT(1, 1)
					break;
				}

				if (dsgcache != NULL &&
				    dsgcache->mark == charset)
					dsg = dsgcache;
				else {
					for (dsg = CONFIG_DESIGNATIONS;
					     dsg->mark != charset
#ifdef Py_DEBUG
						&& dsg->mark != '\0'
#endif
					     ;dsg++)
						/* noop */;
					assert(dsg->mark != '\0');
					dsgcache = dsg;
				}

				REQUIRE_INBUF(dsg->width)
				decoded = dsg->decoder(*inbuf);
				if (decoded == MAP_UNMAPPABLE)
					return dsg->width;

				if (decoded < 0x10000) {
					WRITE1(decoded)
					NEXT_OUT(1)
				}
				else if (decoded < 0x30000) {
					WRITEUCS4(decoded)
				}
				else { /* JIS X 0213 pairs */
					WRITE2(decoded >> 16, decoded & 0xffff)
					NEXT_OUT(2)
				}
				NEXT_IN(dsg->width)
			}
			break;
		}
	}
	return 0;
}

/*-*- mapping table holders -*-*/

#define ENCMAP(enc) static const encode_map *enc##_encmap = NULL;
#define DECMAP(enc) static const decode_map *enc##_decmap = NULL;

/* kr */
ENCMAP(cp949)
DECMAP(ksx1001)

/* jp */
ENCMAP(jisxcommon)
DECMAP(jisx0208)
DECMAP(jisx0212)
ENCMAP(jisx0213_bmp)
DECMAP(jisx0213_1_bmp)
DECMAP(jisx0213_2_bmp)
ENCMAP(jisx0213_emp)
DECMAP(jisx0213_1_emp)
DECMAP(jisx0213_2_emp)

/* cn */
ENCMAP(gbcommon)
DECMAP(gb2312)

/* tw */

/*-*- mapping access functions -*-*/

static int
ksx1001_init(void)
{
	static int initialized = 0;

	if (!initialized && (
			IMPORT_MAP(kr, cp949, &cp949_encmap, NULL) ||
			IMPORT_MAP(kr, ksx1001, NULL, &ksx1001_decmap)))
		return -1;
	initialized = 1;
	return 0;
}

static ucs4_t
ksx1001_decoder(const unsigned char *data)
{
	ucs4_t u;
	TRYMAP_DEC(ksx1001, u, data[0], data[1])
		return u;
	else
		return MAP_UNMAPPABLE;
}

static DBCHAR
ksx1001_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	assert(*length == 1);
	if (*data < 0x10000) {
		TRYMAP_ENC(cp949, coded, *data)
			if (!(coded & 0x8000))
				return coded;
	}
	return MAP_UNMAPPABLE;
}

static int
jisx0208_init(void)
{
	static int initialized = 0;

	if (!initialized && (
			IMPORT_MAP(jp, jisxcommon, &jisxcommon_encmap, NULL) ||
			IMPORT_MAP(jp, jisx0208, NULL, &jisx0208_decmap)))
		return -1;
	initialized = 1;
	return 0;
}

static ucs4_t
jisx0208_decoder(const unsigned char *data)
{
	ucs4_t u;
	if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
		return 0xff3c;
	else TRYMAP_DEC(jisx0208, u, data[0], data[1])
		return u;
	else
		return MAP_UNMAPPABLE;
}

static DBCHAR
jisx0208_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	assert(*length == 1);
	if (*data < 0x10000) {
		if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */
			return 0x2140;
		else TRYMAP_ENC(jisxcommon, coded, *data) {
			if (!(coded & 0x8000))
				return coded;
		}
	}
	return MAP_UNMAPPABLE;
}

static int
jisx0212_init(void)
{
	static int initialized = 0;

	if (!initialized && (
			IMPORT_MAP(jp, jisxcommon, &jisxcommon_encmap, NULL) ||
			IMPORT_MAP(jp, jisx0212, NULL, &jisx0212_decmap)))
		return -1;
	initialized = 1;
	return 0;
}

static ucs4_t
jisx0212_decoder(const unsigned char *data)
{
	ucs4_t u;
	TRYMAP_DEC(jisx0212, u, data[0], data[1])
		return u;
	else
		return MAP_UNMAPPABLE;
}

static DBCHAR
jisx0212_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	assert(*length == 1);
	if (*data < 0x10000) {
		TRYMAP_ENC(jisxcommon, coded, *data) {
			if (coded & 0x8000)
				return coded & 0x7fff;
		}
	}
	return MAP_UNMAPPABLE;
}

static int
jisx0213_init(void)
{
	static int initialized = 0;

	if (!initialized && (
			jisx0208_init() ||
			IMPORT_MAP(jp, jisx0213_bmp,
				   &jisx0213_bmp_encmap, NULL) ||
			IMPORT_MAP(jp, jisx0213_1_bmp,
				   NULL, &jisx0213_1_bmp_decmap) ||
			IMPORT_MAP(jp, jisx0213_2_bmp,
				   NULL, &jisx0213_2_bmp_decmap) ||
			IMPORT_MAP(jp, jisx0213_emp,
				   &jisx0213_emp_encmap, NULL) ||
			IMPORT_MAP(jp, jisx0213_1_emp,
				   NULL, &jisx0213_1_emp_decmap) ||
			IMPORT_MAP(jp, jisx0213_2_emp,
				   NULL, &jisx0213_2_emp_decmap) ||
			IMPORT_MAP(jp, jisx0213_pair, &jisx0213_pair_encmap,
				   &jisx0213_pair_decmap)))
		return -1;
	initialized = 1;
	return 0;
}

#define config ((void *)2000)
static ucs4_t
jisx0213_2000_1_decoder(const unsigned char *data)
{
	ucs4_t u;
	EMULATE_JISX0213_2000_DECODE_PLANE1(u, data[0], data[1])
	else if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
		return 0xff3c;
	else TRYMAP_DEC(jisx0208, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_1_bmp, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_1_emp, u, data[0], data[1])
		u |= 0x20000;
	else TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]);
	else
		return MAP_UNMAPPABLE;
	return u;
}

static ucs4_t
jisx0213_2000_2_decoder(const unsigned char *data)
{
	ucs4_t u;
	EMULATE_JISX0213_2000_DECODE_PLANE2(u, data[0], data[1])
	TRYMAP_DEC(jisx0213_2_bmp, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_2_emp, u, data[0], data[1])
		u |= 0x20000;
	else
		return MAP_UNMAPPABLE;
	return u;
}
#undef config

static ucs4_t
jisx0213_2004_1_decoder(const unsigned char *data)
{
	ucs4_t u;
	if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
		return 0xff3c;
	else TRYMAP_DEC(jisx0208, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_1_bmp, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_1_emp, u, data[0], data[1])
		u |= 0x20000;
	else TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]);
	else
		return MAP_UNMAPPABLE;
	return u;
}

static ucs4_t
jisx0213_2004_2_decoder(const unsigned char *data)
{
	ucs4_t u;
	TRYMAP_DEC(jisx0213_2_bmp, u, data[0], data[1]);
	else TRYMAP_DEC(jisx0213_2_emp, u, data[0], data[1])
		u |= 0x20000;
	else
		return MAP_UNMAPPABLE;
	return u;
}

static DBCHAR
jisx0213_encoder(const ucs4_t *data, Py_ssize_t *length, void *config)
{
	DBCHAR coded;

	switch (*length) {
	case 1: /* first character */
		if (*data >= 0x10000) {
			if ((*data) >> 16 == 0x20000 >> 16) {
				EMULATE_JISX0213_2000_ENCODE_EMP(coded, *data)
				else TRYMAP_ENC(jisx0213_emp, coded,
						(*data) & 0xffff)
					return coded;
			}
			return MAP_UNMAPPABLE;
		}

		EMULATE_JISX0213_2000_ENCODE_BMP(coded, *data)
		else TRYMAP_ENC(jisx0213_bmp, coded, *data) {
			if (coded == MULTIC)
				return MAP_MULTIPLE_AVAIL;
		}
		else TRYMAP_ENC(jisxcommon, coded, *data) {
			if (coded & 0x8000)
				return MAP_UNMAPPABLE;
		}
		else
			return MAP_UNMAPPABLE;
		return coded;
	case 2: /* second character of unicode pair */
		coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1],
				jisx0213_pair_encmap, JISX0213_ENCPAIRS);
		if (coded == DBCINV) {
			*length = 1;
			coded = find_pairencmap((ucs2_t)data[0], 0,
				  jisx0213_pair_encmap, JISX0213_ENCPAIRS);
			if (coded == DBCINV)
				return MAP_UNMAPPABLE;
		}
		else
			return coded;
	case -1: /* flush unterminated */
		*length = 1;
		coded = find_pairencmap((ucs2_t)data[0], 0,
				jisx0213_pair_encmap, JISX0213_ENCPAIRS);
		if (coded == DBCINV)
			return MAP_UNMAPPABLE;
		else
			return coded;
	default:
		return MAP_UNMAPPABLE;
	}
}

static DBCHAR
jisx0213_2000_1_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded = jisx0213_encoder(data, length, (void *)2000);
	if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
		return coded;
	else if (coded & 0x8000)
		return MAP_UNMAPPABLE;
	else
		return coded;
}

static DBCHAR
jisx0213_2000_1_encoder_paironly(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	Py_ssize_t ilength = *length;

	coded = jisx0213_encoder(data, length, (void *)2000);
	switch (ilength) {
	case 1:
		if (coded == MAP_MULTIPLE_AVAIL)
			return MAP_MULTIPLE_AVAIL;
		else
			return MAP_UNMAPPABLE;
	case 2:
		if (*length != 2)
			return MAP_UNMAPPABLE;
		else
			return coded;
	default:
		return MAP_UNMAPPABLE;
	}
}

static DBCHAR
jisx0213_2000_2_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded = jisx0213_encoder(data, length, (void *)2000);
	if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
		return coded;
	else if (coded & 0x8000)
		return coded & 0x7fff;
	else
		return MAP_UNMAPPABLE;
}

static DBCHAR
jisx0213_2004_1_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded = jisx0213_encoder(data, length, NULL);
	if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
		return coded;
	else if (coded & 0x8000)
		return MAP_UNMAPPABLE;
	else
		return coded;
}

static DBCHAR
jisx0213_2004_1_encoder_paironly(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	Py_ssize_t ilength = *length;

	coded = jisx0213_encoder(data, length, NULL);
	switch (ilength) {
	case 1:
		if (coded == MAP_MULTIPLE_AVAIL)
			return MAP_MULTIPLE_AVAIL;
		else
			return MAP_UNMAPPABLE;
	case 2:
		if (*length != 2)
			return MAP_UNMAPPABLE;
		else
			return coded;
	default:
		return MAP_UNMAPPABLE;
	}
}

static DBCHAR
jisx0213_2004_2_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded = jisx0213_encoder(data, length, NULL);
	if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
		return coded;
	else if (coded & 0x8000)
		return coded & 0x7fff;
	else
		return MAP_UNMAPPABLE;
}

static ucs4_t
jisx0201_r_decoder(const unsigned char *data)
{
	ucs4_t u;
	JISX0201_R_DECODE(*data, u)
	else return MAP_UNMAPPABLE;
	return u;
}

static DBCHAR
jisx0201_r_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	JISX0201_R_ENCODE(*data, coded)
	else return MAP_UNMAPPABLE;
	return coded;
}

static ucs4_t
jisx0201_k_decoder(const unsigned char *data)
{
	ucs4_t u;
	JISX0201_K_DECODE(*data ^ 0x80, u)
	else return MAP_UNMAPPABLE;
	return u;
}

static DBCHAR
jisx0201_k_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	JISX0201_K_ENCODE(*data, coded)
	else return MAP_UNMAPPABLE;
	return coded - 0x80;
}

static int
gb2312_init(void)
{
	static int initialized = 0;

	if (!initialized && (
			IMPORT_MAP(cn, gbcommon, &gbcommon_encmap, NULL) ||
			IMPORT_MAP(cn, gb2312, NULL, &gb2312_decmap)))
		return -1;
	initialized = 1;
	return 0;
}

static ucs4_t
gb2312_decoder(const unsigned char *data)
{
	ucs4_t u;
	TRYMAP_DEC(gb2312, u, data[0], data[1])
		return u;
	else
		return MAP_UNMAPPABLE;
}

static DBCHAR
gb2312_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	DBCHAR coded;
	assert(*length == 1);
	if (*data < 0x10000) {
		TRYMAP_ENC(gbcommon, coded, *data) {
			if (!(coded & 0x8000))
				return coded;
		}
	}
	return MAP_UNMAPPABLE;
}


static ucs4_t
dummy_decoder(const unsigned char *data)
{
	return MAP_UNMAPPABLE;
}

static DBCHAR
dummy_encoder(const ucs4_t *data, Py_ssize_t *length)
{
	return MAP_UNMAPPABLE;
}

/*-*- registry tables -*-*/

#define REGISTRY_KSX1001_G0	{ CHARSET_KSX1001, 0, 2,		\
				  ksx1001_init,				\
				  ksx1001_decoder, ksx1001_encoder }
#define REGISTRY_KSX1001_G1	{ CHARSET_KSX1001, 1, 2,		\
				  ksx1001_init,				\
				  ksx1001_decoder, ksx1001_encoder }
#define REGISTRY_JISX0201_R	{ CHARSET_JISX0201_R, 0, 1,		\
				  NULL,					\
				  jisx0201_r_decoder, jisx0201_r_encoder }
#define REGISTRY_JISX0201_K	{ CHARSET_JISX0201_K, 0, 1,		\
				  NULL,					\
				  jisx0201_k_decoder, jisx0201_k_encoder }
#define REGISTRY_JISX0208	{ CHARSET_JISX0208, 0, 2,		\
				  jisx0208_init,			\
				  jisx0208_decoder, jisx0208_encoder }
#define REGISTRY_JISX0208_O	{ CHARSET_JISX0208_O, 0, 2,		\
				  jisx0208_init,			\
				  jisx0208_decoder, jisx0208_encoder }
#define REGISTRY_JISX0212	{ CHARSET_JISX0212, 0, 2,		\
				  jisx0212_init,			\
				  jisx0212_decoder, jisx0212_encoder }
#define REGISTRY_JISX0213_2000_1 { CHARSET_JISX0213_2000_1, 0, 2,	\
				  jisx0213_init,			\
				  jisx0213_2000_1_decoder,		\
				  jisx0213_2000_1_encoder }
#define REGISTRY_JISX0213_2000_1_PAIRONLY { CHARSET_JISX0213_2000_1, 0, 2, \
				  jisx0213_init,			\
				  jisx0213_2000_1_decoder,		\
				  jisx0213_2000_1_encoder_paironly }
#define REGISTRY_JISX0213_2000_2 { CHARSET_JISX0213_2, 0, 2,		\
				  jisx0213_init,			\
				  jisx0213_2000_2_decoder,		\
				  jisx0213_2000_2_encoder }
#define REGISTRY_JISX0213_2004_1 { CHARSET_JISX0213_2004_1, 0, 2,	\
				  jisx0213_init,			\
				  jisx0213_2004_1_decoder,		\
				  jisx0213_2004_1_encoder }
#define REGISTRY_JISX0213_2004_1_PAIRONLY { CHARSET_JISX0213_2004_1, 0, 2, \
				  jisx0213_init,			\
				  jisx0213_2004_1_decoder,		\
				  jisx0213_2004_1_encoder_paironly }
#define REGISTRY_JISX0213_2004_2 { CHARSET_JISX0213_2, 0, 2,		\
				  jisx0213_init,			\
				  jisx0213_2004_2_decoder,		\
				  jisx0213_2004_2_encoder }
#define REGISTRY_GB2312		{ CHARSET_GB2312, 0, 2,			\
				  gb2312_init,				\
				  gb2312_decoder, gb2312_encoder }
#define REGISTRY_CNS11643_1	{ CHARSET_CNS11643_1, 1, 2,		\
				  cns11643_init,			\
				  cns11643_1_decoder, cns11643_1_encoder }
#define REGISTRY_CNS11643_2	{ CHARSET_CNS11643_2, 2, 2,		\
				  cns11643_init,			\
				  cns11643_2_decoder, cns11643_2_encoder }
#define REGISTRY_ISO8859_1	{ CHARSET_ISO8859_1, 2, 1,		\
				  NULL, dummy_decoder, dummy_encoder }
#define REGISTRY_ISO8859_7	{ CHARSET_ISO8859_7, 2, 1,		\
				  NULL, dummy_decoder, dummy_encoder }
#define REGISTRY_SENTINEL	{ 0, }
#define CONFIGDEF(var, attrs)						\
	static const struct iso2022_config iso2022_##var##_config = {	\
		attrs, iso2022_##var##_designations			\
	};

static const struct iso2022_designation iso2022_kr_designations[] = {
	REGISTRY_KSX1001_G1, REGISTRY_SENTINEL
};
CONFIGDEF(kr, 0)

static const struct iso2022_designation iso2022_jp_designations[] = {
	REGISTRY_JISX0208, REGISTRY_JISX0201_R, REGISTRY_JISX0208_O,
	REGISTRY_SENTINEL
};
CONFIGDEF(jp, NO_SHIFT | USE_JISX0208_EXT)

static const struct iso2022_designation iso2022_jp_1_designations[] = {
	REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_JISX0201_R,
	REGISTRY_JISX0208_O, REGISTRY_SENTINEL
};
CONFIGDEF(jp_1, NO_SHIFT | USE_JISX0208_EXT)

static const struct iso2022_designation iso2022_jp_2_designations[] = {
	REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_KSX1001_G0,
	REGISTRY_GB2312, REGISTRY_JISX0201_R, REGISTRY_JISX0208_O,
	REGISTRY_ISO8859_1, REGISTRY_ISO8859_7, REGISTRY_SENTINEL
};
CONFIGDEF(jp_2, NO_SHIFT | USE_G2 | USE_JISX0208_EXT)

static const struct iso2022_designation iso2022_jp_2004_designations[] = {
	REGISTRY_JISX0213_2004_1_PAIRONLY, REGISTRY_JISX0208,
	REGISTRY_JISX0213_2004_1, REGISTRY_JISX0213_2004_2, REGISTRY_SENTINEL
};
CONFIGDEF(jp_2004, NO_SHIFT | USE_JISX0208_EXT)

static const struct iso2022_designation iso2022_jp_3_designations[] = {
	REGISTRY_JISX0213_2000_1_PAIRONLY, REGISTRY_JISX0208,
	REGISTRY_JISX0213_2000_1, REGISTRY_JISX0213_2000_2, REGISTRY_SENTINEL
};
CONFIGDEF(jp_3, NO_SHIFT | USE_JISX0208_EXT)

static const struct iso2022_designation iso2022_jp_ext_designations[] = {
	REGISTRY_JISX0208, REGISTRY_JISX0212, REGISTRY_JISX0201_R,
	REGISTRY_JISX0201_K, REGISTRY_JISX0208_O, REGISTRY_SENTINEL
};
CONFIGDEF(jp_ext, NO_SHIFT | USE_JISX0208_EXT)


BEGIN_MAPPINGS_LIST
  /* no mapping table here */
END_MAPPINGS_LIST

#define ISO2022_CODEC(variation) {		\
	"iso2022_" #variation,			\
	&iso2022_##variation##_config,		\
	iso2022_codec_init,			\
	_STATEFUL_METHODS(iso2022)		\
},

BEGIN_CODECS_LIST
  ISO2022_CODEC(kr)
  ISO2022_CODEC(jp)
  ISO2022_CODEC(jp_1)
  ISO2022_CODEC(jp_2)
  ISO2022_CODEC(jp_2004)
  ISO2022_CODEC(jp_3)
  ISO2022_CODEC(jp_ext)
END_CODECS_LIST

I_AM_A_MODULE_FOR(iso2022)