Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-103583: Isolate CJK codec modules (#103869)

This commit is contained in:
Erlend E. Aasland 2023-04-27 15:02:43 +02:00 committed by GitHub
parent e9c777208f
commit 8a0c7f1e40
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 228 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS.d/next/Library/2023-04-26-15-14-36.gh-issue-103583.iCMDFt.rst Normal file
View File

@ -0,0 +1,2 @@
Isolate :mod:`!_multibytecodec` and codecs extension modules. Patches by
Erlend E. Aasland.

18
Modules/cjkcodecs/_codecs_hk.c
View File

@ -6,6 +6,10 @@
#define USING_IMPORTED_MAPS #define USING_IMPORTED_MAPS
#define CJK_MOD_SPECIFIC_STATE \
const encode_map *big5_encmap; \
const decode_map *big5_decmap;
#include "cjkcodecs.h" #include "cjkcodecs.h"
#include "mappings_hk.h" #include "mappings_hk.h"
@ -13,16 +17,12 @@
* BIG5HKSCS codec * BIG5HKSCS codec
*/ */
static const encode_map *big5_encmap = NULL;
static const decode_map *big5_decmap = NULL;
CODEC_INIT(big5hkscs) CODEC_INIT(big5hkscs)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (IMPORT_MAP(tw, big5, &st->big5_encmap, &st->big5_decmap)) {
if (!initialized && IMPORT_MAP(tw, big5, &big5_encmap, &big5_decmap))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
@ -81,7 +81,7 @@ ENCODER(big5hkscs)
} }
} }
} }
else if (TRYMAP_ENC(big5, code, c)) else if (TRYMAP_ENC_ST(big5, code, c))
; ;
else else
return 1; return 1;
@ -122,7 +122,7 @@ DECODER(big5hkscs)
REQUIRE_INBUF(2); REQUIRE_INBUF(2);
if (0xc6 > c || c > 0xc8 || (c < 0xc7 && INBYTE2 < 0xa1)) { if (0xc6 > c || c > 0xc8 || (c < 0xc7 && INBYTE2 < 0xa1)) {
if (TRYMAP_DEC(big5, decoded, c, INBYTE2)) { if (TRYMAP_DEC_ST(big5, decoded, c, INBYTE2)) {
OUTCHAR(decoded); OUTCHAR(decoded);
NEXT_IN(2); NEXT_IN(2);
continue; continue;

289
Modules/cjkcodecs/_codecs_iso2022.c
View File

@ -10,6 +10,27 @@
#define EMULATE_JISX0213_2000_ENCODE_INVALID MAP_UNMAPPABLE #define EMULATE_JISX0213_2000_ENCODE_INVALID MAP_UNMAPPABLE
#define EMULATE_JISX0213_2000_DECODE_INVALID MAP_UNMAPPABLE #define EMULATE_JISX0213_2000_DECODE_INVALID MAP_UNMAPPABLE
#define CJK_MOD_SPECIFIC_STATE \
/* kr */ \
const encode_map *cp949_encmap; \
const decode_map *ksx1001_decmap; \
\
/* jp */ \
const encode_map *jisxcommon_encmap; \
const decode_map *jisx0208_decmap; \
const decode_map *jisx0212_decmap; \
const encode_map *jisx0213_bmp_encmap; \
const decode_map *jisx0213_1_bmp_decmap; \
const decode_map *jisx0213_2_bmp_decmap; \
const encode_map *jisx0213_emp_encmap; \
const decode_map *jisx0213_1_emp_decmap; \
const decode_map *jisx0213_2_emp_decmap; \
\
/* cn */ \
const encode_map *gbcommon_encmap; \
const decode_map *gb2312_decmap;
#include "cjkcodecs.h" #include "cjkcodecs.h"
#include "alg_jisx0201.h" #include "alg_jisx0201.h"
#include "emu_jisx0213_2000.h" #include "emu_jisx0213_2000.h"
@ -90,7 +111,7 @@
#define STATE_CLEARFLAG(f) do { ((state)->c[4]) &= ~(f); } while (0) #define STATE_CLEARFLAG(f) do { ((state)->c[4]) &= ~(f); } while (0)
#define STATE_CLEARFLAGS() do { ((state)->c[4]) = 0; } while (0) #define STATE_CLEARFLAGS() do { ((state)->c[4]) = 0; } while (0)
#define ISO2022_CONFIG ((const struct iso2022_config *)config) #define ISO2022_CONFIG ((const struct iso2022_config *)(codec->config))
#define CONFIG_ISSET(flag) (ISO2022_CONFIG->flags & (flag)) #define CONFIG_ISSET(flag) (ISO2022_CONFIG->flags & (flag))
#define CONFIG_DESIGNATIONS (ISO2022_CONFIG->designations) #define CONFIG_DESIGNATIONS (ISO2022_CONFIG->designations)
@ -101,9 +122,12 @@
/*-*- internal data structures -*-*/ /*-*- internal data structures -*-*/
typedef int (*iso2022_init_func)(void); typedef int (*iso2022_init_func)(const MultibyteCodec *codec);
typedef Py_UCS4 (*iso2022_decode_func)(const unsigned char *data); typedef Py_UCS4 (*iso2022_decode_func)(const MultibyteCodec *codec,
typedef DBCHAR (*iso2022_encode_func)(const Py_UCS4 *data, Py_ssize_t *length); const unsigned char *data);
typedef DBCHAR (*iso2022_encode_func)(const MultibyteCodec *codec,
const Py_UCS4 *data,
Py_ssize_t *length);
struct iso2022_designation { struct iso2022_designation {
unsigned char mark; unsigned char mark;
@ -124,9 +148,11 @@ struct iso2022_config {
CODEC_INIT(iso2022) CODEC_INIT(iso2022)
{ {
const struct iso2022_designation *desig; const struct iso2022_designation *desig;
for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++) for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++) {
if (desig->initializer != NULL && desig->initializer() != 0) if (desig->initializer != NULL && desig->initializer(codec) != 0) {
return -1; return -1;
}
}
return 0; return 0;
} }
@ -182,7 +208,7 @@ ENCODER(iso2022)
encoded = MAP_UNMAPPABLE; encoded = MAP_UNMAPPABLE;
for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) { for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
Py_ssize_t length = 1; Py_ssize_t length = 1;
encoded = dsg->encoder(&c, &length); encoded = dsg->encoder(codec, &c, &length);
if (encoded == MAP_MULTIPLE_AVAIL) { if (encoded == MAP_MULTIPLE_AVAIL) {
/* this implementation won't work for pair /* this implementation won't work for pair
* of non-bmp characters. */ * of non-bmp characters. */
@ -193,7 +219,7 @@ ENCODER(iso2022)
} }
else else
length = 2; length = 2;
encoded = dsg->encoder(&c, &length); encoded = dsg->encoder(codec, &c, &length);
if (encoded != MAP_UNMAPPABLE) { if (encoded != MAP_UNMAPPABLE) {
insize = length; insize = length;
break; break;
@ -288,7 +314,7 @@ DECODER_RESET(iso2022)
} }
static Py_ssize_t static Py_ssize_t
iso2022processesc(const void *config, MultibyteCodec_State *state, iso2022processesc(const MultibyteCodec *codec, MultibyteCodec_State *state,
const unsigned char **inbuf, Py_ssize_t *inleft) const unsigned char **inbuf, Py_ssize_t *inleft)
{ {
unsigned char charset, designation; unsigned char charset, designation;
@ -388,7 +414,7 @@ iso2022processesc(const void *config, MultibyteCodec_State *state,
} }
static Py_ssize_t static Py_ssize_t
iso2022processg2(const void *config, MultibyteCodec_State *state, iso2022processg2(const MultibyteCodec *codec, MultibyteCodec_State *state,
const unsigned char **inbuf, Py_ssize_t *inleft, const unsigned char **inbuf, Py_ssize_t *inleft,
_PyUnicodeWriter *writer) _PyUnicodeWriter *writer)
{ {
@ -442,14 +468,14 @@ DECODER(iso2022)
case ESC: case ESC:
REQUIRE_INBUF(2); REQUIRE_INBUF(2);
if (IS_ISO2022ESC(INBYTE2)) { if (IS_ISO2022ESC(INBYTE2)) {
err = iso2022processesc(config, state, err = iso2022processesc(codec, state,
inbuf, &inleft); inbuf, &inleft);
if (err != 0) if (err != 0)
return err; return err;
} }
else if (CONFIG_ISSET(USE_G2) && INBYTE2 == 'N') {/* SS2 */ else if (CONFIG_ISSET(USE_G2) && INBYTE2 == 'N') {/* SS2 */
REQUIRE_INBUF(3); REQUIRE_INBUF(3);
err = iso2022processg2(config, state, err = iso2022processg2(codec, state,
inbuf, &inleft, writer); inbuf, &inleft, writer);
if (err != 0) if (err != 0)
return err; return err;
@ -517,7 +543,7 @@ bypass:
} }
REQUIRE_INBUF(dsg->width); REQUIRE_INBUF(dsg->width);
decoded = dsg->decoder(*inbuf); decoded = dsg->decoder(codec, *inbuf);
if (decoded == MAP_UNMAPPABLE) if (decoded == MAP_UNMAPPABLE)
return dsg->width; return dsg->width;
@ -538,64 +564,38 @@ bypass:
return 0; return 0;
} }
/*-*- mapping table holders -*-*/
#define ENCMAP(enc) static const encode_map *enc##_encmap = NULL;
#define DECMAP(enc) static const decode_map *enc##_decmap = NULL;
/* kr */
ENCMAP(cp949)
DECMAP(ksx1001)
/* jp */
ENCMAP(jisxcommon)
DECMAP(jisx0208)
DECMAP(jisx0212)
ENCMAP(jisx0213_bmp)
DECMAP(jisx0213_1_bmp)
DECMAP(jisx0213_2_bmp)
ENCMAP(jisx0213_emp)
DECMAP(jisx0213_1_emp)
DECMAP(jisx0213_2_emp)
/* cn */
ENCMAP(gbcommon)
DECMAP(gb2312)
/* tw */
/*-*- mapping access functions -*-*/ /*-*- mapping access functions -*-*/
static int static int
ksx1001_init(void) ksx1001_init(const MultibyteCodec *codec)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (IMPORT_MAP(kr, cp949, &st->cp949_encmap, NULL) ||
if (!initialized && ( IMPORT_MAP(kr, ksx1001, NULL, &st->ksx1001_decmap))
IMPORT_MAP(kr, cp949, &cp949_encmap, NULL) || {
IMPORT_MAP(kr, ksx1001, NULL, &ksx1001_decmap)))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
static Py_UCS4 static Py_UCS4
ksx1001_decoder(const unsigned char *data) ksx1001_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (TRYMAP_DEC(ksx1001, u, data[0], data[1])) if (TRYMAP_DEC_ST(ksx1001, u, data[0], data[1]))
return u; return u;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
static DBCHAR static DBCHAR
ksx1001_encoder(const Py_UCS4 *data, Py_ssize_t *length) ksx1001_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
assert(*length == 1); assert(*length == 1);
if (*data < 0x10000) { if (*data < 0x10000) {
if (TRYMAP_ENC(cp949, coded, *data)) { if (TRYMAP_ENC_ST(cp949, coded, *data)) {
if (!(coded & 0x8000)) if (!(coded & 0x8000))
return coded; return coded;
} }
@ -604,39 +604,39 @@ ksx1001_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static int static int
jisx0208_init(void) jisx0208_init(const MultibyteCodec *codec)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (IMPORT_MAP(jp, jisxcommon, &st->jisxcommon_encmap, NULL) ||
if (!initialized && ( IMPORT_MAP(jp, jisx0208, NULL, &st->jisx0208_decmap))
IMPORT_MAP(jp, jisxcommon, &jisxcommon_encmap, NULL) || {
IMPORT_MAP(jp, jisx0208, NULL, &jisx0208_decmap)))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
static Py_UCS4 static Py_UCS4
jisx0208_decoder(const unsigned char *data) jisx0208_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */ if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
return 0xff3c; return 0xff3c;
else if (TRYMAP_DEC(jisx0208, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
return u; return u;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
static DBCHAR static DBCHAR
jisx0208_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0208_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
assert(*length == 1); assert(*length == 1);
if (*data < 0x10000) { if (*data < 0x10000) {
if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */ if (*data == 0xff3c) /* F/W REVERSE SOLIDUS */
return 0x2140; return 0x2140;
else if (TRYMAP_ENC(jisxcommon, coded, *data)) { else if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
if (!(coded & 0x8000)) if (!(coded & 0x8000))
return coded; return coded;
} }
@ -645,35 +645,35 @@ jisx0208_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static int static int
jisx0212_init(void) jisx0212_init(const MultibyteCodec *codec)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (IMPORT_MAP(jp, jisxcommon, &st->jisxcommon_encmap, NULL) ||
if (!initialized && ( IMPORT_MAP(jp, jisx0212, NULL, &st->jisx0212_decmap))
IMPORT_MAP(jp, jisxcommon, &jisxcommon_encmap, NULL) || {
IMPORT_MAP(jp, jisx0212, NULL, &jisx0212_decmap)))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
static Py_UCS4 static Py_UCS4
jisx0212_decoder(const unsigned char *data) jisx0212_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (TRYMAP_DEC(jisx0212, u, data[0], data[1])) if (TRYMAP_DEC_ST(jisx0212, u, data[0], data[1]))
return u; return u;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
static DBCHAR static DBCHAR
jisx0212_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0212_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
assert(*length == 1); assert(*length == 1);
if (*data < 0x10000) { if (*data < 0x10000) {
if (TRYMAP_ENC(jisxcommon, coded, *data)) { if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
if (coded & 0x8000) if (coded & 0x8000)
return coded & 0x7fff; return coded & 0x7fff;
} }
@ -682,44 +682,37 @@ jisx0212_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static int static int
jisx0213_init(void) jisx0213_init(const MultibyteCodec *codec)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (jisx0208_init(codec) ||
if (!initialized && ( IMPORT_MAP(jp, jisx0213_bmp, &st->jisx0213_bmp_encmap, NULL) ||
jisx0208_init() || IMPORT_MAP(jp, jisx0213_1_bmp, NULL, &st->jisx0213_1_bmp_decmap) ||
IMPORT_MAP(jp, jisx0213_bmp, IMPORT_MAP(jp, jisx0213_2_bmp, NULL, &st->jisx0213_2_bmp_decmap) ||
&jisx0213_bmp_encmap, NULL) || IMPORT_MAP(jp, jisx0213_emp, &st->jisx0213_emp_encmap, NULL) ||
IMPORT_MAP(jp, jisx0213_1_bmp, IMPORT_MAP(jp, jisx0213_1_emp, NULL, &st->jisx0213_1_emp_decmap) ||
NULL, &jisx0213_1_bmp_decmap) || IMPORT_MAP(jp, jisx0213_2_emp, NULL, &st->jisx0213_2_emp_decmap) ||
IMPORT_MAP(jp, jisx0213_2_bmp, IMPORT_MAP(jp, jisx0213_pair,
NULL, &jisx0213_2_bmp_decmap) || &jisx0213_pair_encmap, &jisx0213_pair_decmap))
IMPORT_MAP(jp, jisx0213_emp, {
&jisx0213_emp_encmap, NULL) ||
IMPORT_MAP(jp, jisx0213_1_emp,
NULL, &jisx0213_1_emp_decmap) ||
IMPORT_MAP(jp, jisx0213_2_emp,
NULL, &jisx0213_2_emp_decmap) ||
IMPORT_MAP(jp, jisx0213_pair, &jisx0213_pair_encmap,
&jisx0213_pair_decmap)))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
#define config ((void *)2000) #define config ((void *)2000)
static Py_UCS4 static Py_UCS4
jisx0213_2000_1_decoder(const unsigned char *data) jisx0213_2000_1_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
EMULATE_JISX0213_2000_DECODE_PLANE1(u, data[0], data[1]) EMULATE_JISX0213_2000_DECODE_PLANE1(config, u, data[0], data[1])
else if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */ else if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
return 0xff3c; return 0xff3c;
else if (TRYMAP_DEC(jisx0208, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_1_bmp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_1_bmp, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_1_emp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_1_emp, u, data[0], data[1]))
u |= 0x20000; u |= 0x20000;
else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1])) else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]))
; ;
@ -729,13 +722,13 @@ jisx0213_2000_1_decoder(const unsigned char *data)
} }
static Py_UCS4 static Py_UCS4
jisx0213_2000_2_decoder(const unsigned char *data) jisx0213_2000_2_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(u, data[0], data[1]) EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(config, u, data[0], data[1])
if (TRYMAP_DEC(jisx0213_2_bmp, u, data[0], data[1])) if (TRYMAP_DEC_ST(jisx0213_2_bmp, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_2_emp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_2_emp, u, data[0], data[1]))
u |= 0x20000; u |= 0x20000;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
@ -744,16 +737,16 @@ jisx0213_2000_2_decoder(const unsigned char *data)
#undef config #undef config
static Py_UCS4 static Py_UCS4
jisx0213_2004_1_decoder(const unsigned char *data) jisx0213_2004_1_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */ if (data[0] == 0x21 && data[1] == 0x40) /* F/W REVERSE SOLIDUS */
return 0xff3c; return 0xff3c;
else if (TRYMAP_DEC(jisx0208, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0208, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_1_bmp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_1_bmp, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_1_emp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_1_emp, u, data[0], data[1]))
u |= 0x20000; u |= 0x20000;
else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1])) else if (TRYMAP_DEC(jisx0213_pair, u, data[0], data[1]))
; ;
@ -763,12 +756,12 @@ jisx0213_2004_1_decoder(const unsigned char *data)
} }
static Py_UCS4 static Py_UCS4
jisx0213_2004_2_decoder(const unsigned char *data) jisx0213_2004_2_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (TRYMAP_DEC(jisx0213_2_bmp, u, data[0], data[1])) if (TRYMAP_DEC_ST(jisx0213_2_bmp, u, data[0], data[1]))
; ;
else if (TRYMAP_DEC(jisx0213_2_emp, u, data[0], data[1])) else if (TRYMAP_DEC_ST(jisx0213_2_emp, u, data[0], data[1]))
u |= 0x20000; u |= 0x20000;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
@ -776,7 +769,8 @@ jisx0213_2004_2_decoder(const unsigned char *data)
} }
static DBCHAR static DBCHAR
jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length, void *config) jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length, const void *config)
{ {
DBCHAR coded; DBCHAR coded;
@ -784,19 +778,19 @@ jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length, void *config)
case 1: /* first character */ case 1: /* first character */
if (*data >= 0x10000) { if (*data >= 0x10000) {
if ((*data) >> 16 == 0x20000 >> 16) { if ((*data) >> 16 == 0x20000 >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(coded, *data) EMULATE_JISX0213_2000_ENCODE_EMP(config, coded, *data)
else if (TRYMAP_ENC(jisx0213_emp, coded, (*data) & 0xffff)) else if (TRYMAP_ENC_ST(jisx0213_emp, coded, (*data) & 0xffff))
return coded; return coded;
} }
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
EMULATE_JISX0213_2000_ENCODE_BMP(coded, *data) EMULATE_JISX0213_2000_ENCODE_BMP(config, coded, *data)
else if (TRYMAP_ENC(jisx0213_bmp, coded, *data)) { else if (TRYMAP_ENC_ST(jisx0213_bmp, coded, *data)) {
if (coded == MULTIC) if (coded == MULTIC)
return MAP_MULTIPLE_AVAIL; return MAP_MULTIPLE_AVAIL;
} }
else if (TRYMAP_ENC(jisxcommon, coded, *data)) { else if (TRYMAP_ENC_ST(jisxcommon, coded, *data)) {
if (coded & 0x8000) if (coded & 0x8000)
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
@ -827,9 +821,10 @@ jisx0213_encoder(const Py_UCS4 *data, Py_ssize_t *length, void *config)
} }
static DBCHAR static DBCHAR
jisx0213_2000_1_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2000_1_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded = jisx0213_encoder(data, length, (void *)2000); DBCHAR coded = jisx0213_encoder(codec, data, length, (void *)2000);
if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
return coded; return coded;
else if (coded & 0x8000) else if (coded & 0x8000)
@ -839,12 +834,13 @@ jisx0213_2000_1_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static DBCHAR static DBCHAR
jisx0213_2000_1_encoder_paironly(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2000_1_encoder_paironly(const MultibyteCodec *codec,
const Py_UCS4 *data, Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
Py_ssize_t ilength = *length; Py_ssize_t ilength = *length;
coded = jisx0213_encoder(data, length, (void *)2000); coded = jisx0213_encoder(codec, data, length, (void *)2000);
switch (ilength) { switch (ilength) {
case 1: case 1:
if (coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_MULTIPLE_AVAIL)
@ -862,9 +858,10 @@ jisx0213_2000_1_encoder_paironly(const Py_UCS4 *data, Py_ssize_t *length)
} }
static DBCHAR static DBCHAR
jisx0213_2000_2_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2000_2_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded = jisx0213_encoder(data, length, (void *)2000); DBCHAR coded = jisx0213_encoder(codec, data, length, (void *)2000);
if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
return coded; return coded;
else if (coded & 0x8000) else if (coded & 0x8000)
@ -874,9 +871,10 @@ jisx0213_2000_2_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static DBCHAR static DBCHAR
jisx0213_2004_1_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2004_1_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded = jisx0213_encoder(data, length, NULL); DBCHAR coded = jisx0213_encoder(codec, data, length, NULL);
if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
return coded; return coded;
else if (coded & 0x8000) else if (coded & 0x8000)
@ -886,12 +884,13 @@ jisx0213_2004_1_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static DBCHAR static DBCHAR
jisx0213_2004_1_encoder_paironly(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2004_1_encoder_paironly(const MultibyteCodec *codec,
const Py_UCS4 *data, Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
Py_ssize_t ilength = *length; Py_ssize_t ilength = *length;
coded = jisx0213_encoder(data, length, NULL); coded = jisx0213_encoder(codec, data, length, NULL);
switch (ilength) { switch (ilength) {
case 1: case 1:
if (coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_MULTIPLE_AVAIL)
@ -909,9 +908,10 @@ jisx0213_2004_1_encoder_paironly(const Py_UCS4 *data, Py_ssize_t *length)
} }
static DBCHAR static DBCHAR
jisx0213_2004_2_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0213_2004_2_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded = jisx0213_encoder(data, length, NULL); DBCHAR coded = jisx0213_encoder(codec, data, length, NULL);
if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL) if (coded == MAP_UNMAPPABLE || coded == MAP_MULTIPLE_AVAIL)
return coded; return coded;
else if (coded & 0x8000) else if (coded & 0x8000)
@ -921,7 +921,7 @@ jisx0213_2004_2_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static Py_UCS4 static Py_UCS4
jisx0201_r_decoder(const unsigned char *data) jisx0201_r_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
JISX0201_R_DECODE_CHAR(*data, u) JISX0201_R_DECODE_CHAR(*data, u)
@ -931,7 +931,8 @@ jisx0201_r_decoder(const unsigned char *data)
} }
static DBCHAR static DBCHAR
jisx0201_r_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0201_r_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
JISX0201_R_ENCODE(*data, coded) JISX0201_R_ENCODE(*data, coded)
@ -941,7 +942,7 @@ jisx0201_r_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static Py_UCS4 static Py_UCS4
jisx0201_k_decoder(const unsigned char *data) jisx0201_k_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
JISX0201_K_DECODE_CHAR(*data ^ 0x80, u) JISX0201_K_DECODE_CHAR(*data ^ 0x80, u)
@ -951,7 +952,8 @@ jisx0201_k_decoder(const unsigned char *data)
} }
static DBCHAR static DBCHAR
jisx0201_k_encoder(const Py_UCS4 *data, Py_ssize_t *length) jisx0201_k_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
JISX0201_K_ENCODE(*data, coded) JISX0201_K_ENCODE(*data, coded)
@ -961,35 +963,35 @@ jisx0201_k_encoder(const Py_UCS4 *data, Py_ssize_t *length)
} }
static int static int
gb2312_init(void) gb2312_init(const MultibyteCodec *codec)
{ {
static int initialized = 0; cjkcodecs_module_state *st = codec->modstate;
if (IMPORT_MAP(cn, gbcommon, &st->gbcommon_encmap, NULL) ||
if (!initialized && ( IMPORT_MAP(cn, gb2312, NULL, &st->gb2312_decmap))
IMPORT_MAP(cn, gbcommon, &gbcommon_encmap, NULL) || {
IMPORT_MAP(cn, gb2312, NULL, &gb2312_decmap)))
return -1; return -1;
initialized = 1; }
return 0; return 0;
} }
static Py_UCS4 static Py_UCS4
gb2312_decoder(const unsigned char *data) gb2312_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
Py_UCS4 u; Py_UCS4 u;
if (TRYMAP_DEC(gb2312, u, data[0], data[1])) if (TRYMAP_DEC_ST(gb2312, u, data[0], data[1]))
return u; return u;
else else
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
static DBCHAR static DBCHAR
gb2312_encoder(const Py_UCS4 *data, Py_ssize_t *length) gb2312_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
DBCHAR coded; DBCHAR coded;
assert(*length == 1); assert(*length == 1);
if (*data < 0x10000) { if (*data < 0x10000) {
if (TRYMAP_ENC(gbcommon, coded, *data)) { if (TRYMAP_ENC_ST(gbcommon, coded, *data)) {
if (!(coded & 0x8000)) if (!(coded & 0x8000))
return coded; return coded;
} }
@ -999,13 +1001,14 @@ gb2312_encoder(const Py_UCS4 *data, Py_ssize_t *length)
static Py_UCS4 static Py_UCS4
dummy_decoder(const unsigned char *data) dummy_decoder(const MultibyteCodec *codec, const unsigned char *data)
{ {
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }
static DBCHAR static DBCHAR
dummy_encoder(const Py_UCS4 *data, Py_ssize_t *length) dummy_encoder(const MultibyteCodec *codec, const Py_UCS4 *data,
Py_ssize_t *length)
{ {
return MAP_UNMAPPABLE; return MAP_UNMAPPABLE;
} }

16
Modules/cjkcodecs/_codecs_jp.c
View File

@ -164,7 +164,7 @@ ENCODER(euc_jis_2004)
insize = 1; insize = 1;
if (c <= 0xFFFF) { if (c <= 0xFFFF) {
EMULATE_JISX0213_2000_ENCODE_BMP(code, c) EMULATE_JISX0213_2000_ENCODE_BMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_bmp, code, c)) { else if (TRYMAP_ENC(jisx0213_bmp, code, c)) {
if (code == MULTIC) { if (code == MULTIC) {
if (inlen - *inpos < 2) { if (inlen - *inpos < 2) {
@ -215,7 +215,7 @@ ENCODER(euc_jis_2004)
return 1; return 1;
} }
else if (c >> 16 == EMPBASE >> 16) { else if (c >> 16 == EMPBASE >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(code, c) EMULATE_JISX0213_2000_ENCODE_EMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_emp, code, c & 0xffff)) else if (TRYMAP_ENC(jisx0213_emp, code, c & 0xffff))
; ;
else else
@ -271,7 +271,7 @@ DECODER(euc_jis_2004)
c3 = INBYTE3 ^ 0x80; c3 = INBYTE3 ^ 0x80;
/* JIS X 0213 Plane 2 or JIS X 0212 (see NOTES) */ /* JIS X 0213 Plane 2 or JIS X 0212 (see NOTES) */
EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c2, c3) EMULATE_JISX0213_2000_DECODE_PLANE2(codec->config, writer, c2, c3)
else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c2, c3)) else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c2, c3))
OUTCHAR(decoded); OUTCHAR(decoded);
else if (TRYMAP_DEC(jisx0213_2_emp, code, c2, c3)) { else if (TRYMAP_DEC(jisx0213_2_emp, code, c2, c3)) {
@ -293,7 +293,7 @@ DECODER(euc_jis_2004)
c2 = INBYTE2 ^ 0x80; c2 = INBYTE2 ^ 0x80;
/* JIS X 0213 Plane 1 */ /* JIS X 0213 Plane 1 */
EMULATE_JISX0213_2000_DECODE_PLANE1(writer, c, c2) EMULATE_JISX0213_2000_DECODE_PLANE1(codec->config, writer, c, c2)
else if (c == 0x21 && c2 == 0x40) else if (c == 0x21 && c2 == 0x40)
OUTCHAR(0xff3c); OUTCHAR(0xff3c);
else if (c == 0x22 && c2 == 0x32) else if (c == 0x22 && c2 == 0x32)
@ -582,7 +582,7 @@ ENCODER(shift_jis_2004)
if (code == NOCHAR) { if (code == NOCHAR) {
if (c <= 0xffff) { if (c <= 0xffff) {
EMULATE_JISX0213_2000_ENCODE_BMP(code, c) EMULATE_JISX0213_2000_ENCODE_BMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_bmp, code, c)) { else if (TRYMAP_ENC(jisx0213_bmp, code, c)) {
if (code == MULTIC) { if (code == MULTIC) {
if (inlen - *inpos < 2) { if (inlen - *inpos < 2) {
@ -625,7 +625,7 @@ ENCODER(shift_jis_2004)
return 1; return 1;
} }
else if (c >> 16 == EMPBASE >> 16) { else if (c >> 16 == EMPBASE >> 16) {
EMULATE_JISX0213_2000_ENCODE_EMP(code, c) EMULATE_JISX0213_2000_ENCODE_EMP(codec->config, code, c)
else if (TRYMAP_ENC(jisx0213_emp, code, c&0xffff)) else if (TRYMAP_ENC(jisx0213_emp, code, c&0xffff))
; ;
else else
@ -686,7 +686,7 @@ DECODER(shift_jis_2004)
if (c1 < 0x5e) { /* Plane 1 */ if (c1 < 0x5e) { /* Plane 1 */
c1 += 0x21; c1 += 0x21;
EMULATE_JISX0213_2000_DECODE_PLANE1(writer, EMULATE_JISX0213_2000_DECODE_PLANE1(codec->config, writer,
c1, c2) c1, c2)
else if (TRYMAP_DEC(jisx0208, decoded, c1, c2)) else if (TRYMAP_DEC(jisx0208, decoded, c1, c2))
OUTCHAR(decoded); OUTCHAR(decoded);
@ -708,7 +708,7 @@ DECODER(shift_jis_2004)
else else
c1 -= 0x3d; c1 -= 0x3d;
EMULATE_JISX0213_2000_DECODE_PLANE2(writer, EMULATE_JISX0213_2000_DECODE_PLANE2(codec->config, writer,
c1, c2) c1, c2)
else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c1, c2)) else if (TRYMAP_DEC(jisx0213_2_bmp, decoded, c1, c2))
OUTCHAR(decoded); OUTCHAR(decoded);

36
Modules/cjkcodecs/cjkcodecs.h
View File

@ -60,11 +60,17 @@ struct pair_encodemap {
DBCHAR code; DBCHAR code;
}; };
typedef struct { #ifndef CJK_MOD_SPECIFIC_STATE
#define CJK_MOD_SPECIFIC_STATE
#endif
typedef struct _cjk_mod_state {
int num_mappings; int num_mappings;
int num_codecs; int num_codecs;
struct dbcs_map *mapping_list; struct dbcs_map *mapping_list;
MultibyteCodec *codec_list; MultibyteCodec *codec_list;
CJK_MOD_SPECIFIC_STATE
} cjkcodecs_module_state; } cjkcodecs_module_state;
static inline cjkcodecs_module_state * static inline cjkcodecs_module_state *
@ -76,33 +82,33 @@ get_module_state(PyObject *mod)
} }
#define CODEC_INIT(encoding) \ #define CODEC_INIT(encoding) \
static int encoding##_codec_init(const void *config) static int encoding##_codec_init(const MultibyteCodec *codec)
#define ENCODER_INIT(encoding) \ #define ENCODER_INIT(encoding) \
static int encoding##_encode_init( \ static int encoding##_encode_init( \
MultibyteCodec_State *state, const void *config) MultibyteCodec_State *state, const MultibyteCodec *codec)
#define ENCODER(encoding) \ #define ENCODER(encoding) \
static Py_ssize_t encoding##_encode( \ static Py_ssize_t encoding##_encode( \
MultibyteCodec_State *state, const void *config, \ MultibyteCodec_State *state, const MultibyteCodec *codec, \
int kind, const void *data, \ int kind, const void *data, \
Py_ssize_t *inpos, Py_ssize_t inlen, \ Py_ssize_t *inpos, Py_ssize_t inlen, \
unsigned char **outbuf, Py_ssize_t outleft, int flags) unsigned char **outbuf, Py_ssize_t outleft, int flags)
#define ENCODER_RESET(encoding) \ #define ENCODER_RESET(encoding) \
static Py_ssize_t encoding##_encode_reset( \ static Py_ssize_t encoding##_encode_reset( \
MultibyteCodec_State *state, const void *config, \ MultibyteCodec_State *state, const MultibyteCodec *codec, \
unsigned char **outbuf, Py_ssize_t outleft) unsigned char **outbuf, Py_ssize_t outleft)
#define DECODER_INIT(encoding) \ #define DECODER_INIT(encoding) \
static int encoding##_decode_init( \ static int encoding##_decode_init( \
MultibyteCodec_State *state, const void *config) MultibyteCodec_State *state, const MultibyteCodec *codec)
#define DECODER(encoding) \ #define DECODER(encoding) \
static Py_ssize_t encoding##_decode( \ static Py_ssize_t encoding##_decode( \
MultibyteCodec_State *state, const void *config, \ MultibyteCodec_State *state, const MultibyteCodec *codec, \
const unsigned char **inbuf, Py_ssize_t inleft, \ const unsigned char **inbuf, Py_ssize_t inleft, \
_PyUnicodeWriter *writer) _PyUnicodeWriter *writer)
#define DECODER_RESET(encoding) \ #define DECODER_RESET(encoding) \
static Py_ssize_t encoding##_decode_reset( \ static Py_ssize_t encoding##_decode_reset( \
MultibyteCodec_State *state, const void *config) MultibyteCodec_State *state, const MultibyteCodec *codec)
#define NEXT_IN(i) \ #define NEXT_IN(i) \
do { \ do { \
@ -205,6 +211,9 @@ get_module_state(PyObject *mod)
(m)->bottom]) != NOCHAR) (m)->bottom]) != NOCHAR)
#define TRYMAP_ENC(charset, assi, uni) \ #define TRYMAP_ENC(charset, assi, uni) \
_TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff) _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
#define TRYMAP_ENC_ST(charset, assi, uni) \
_TRYMAP_ENC(&(codec->modstate->charset##_encmap)[(uni) >> 8], \
assi, (uni) & 0xff)
#define _TRYMAP_DEC(m, assi, val) \ #define _TRYMAP_DEC(m, assi, val) \
((m)->map != NULL && \ ((m)->map != NULL && \
@ -213,6 +222,8 @@ get_module_state(PyObject *mod)
((assi) = (m)->map[(val) - (m)->bottom]) != UNIINV) ((assi) = (m)->map[(val) - (m)->bottom]) != UNIINV)
#define TRYMAP_DEC(charset, assi, c1, c2) \ #define TRYMAP_DEC(charset, assi, c1, c2) \
_TRYMAP_DEC(&charset##_decmap[c1], assi, c2) _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
#define TRYMAP_DEC_ST(charset, assi, c1, c2) \
_TRYMAP_DEC(&(codec->modstate->charset##_decmap)[c1], assi, c2)
#define BEGIN_MAPPINGS_LIST(NUM) \ #define BEGIN_MAPPINGS_LIST(NUM) \
static int \ static int \
@ -271,9 +282,12 @@ add_codecs(cjkcodecs_module_state *st) \
#define CODEC_STATELESS_WINIT(enc) \ #define CODEC_STATELESS_WINIT(enc) \
NEXT_CODEC = (MultibyteCodec){#enc, NULL, enc##_codec_init, _STATELESS_METHODS(enc)}; NEXT_CODEC = (MultibyteCodec){#enc, NULL, enc##_codec_init, _STATELESS_METHODS(enc)};
#define END_CODECS_LIST \ #define END_CODECS_LIST \
assert(st->num_codecs == idx); \ assert(st->num_codecs == idx); \
return 0; \ for (int i = 0; i < st->num_codecs; i++) { \
st->codec_list[i].modstate = st; \
} \
return 0; \
} }

22
Modules/cjkcodecs/emu_jisx0213_2000.h
View File

@ -5,8 +5,8 @@
# define EMULATE_JISX0213_2000_ENCODE_INVALID 1 # define EMULATE_JISX0213_2000_ENCODE_INVALID 1
#endif #endif
#define EMULATE_JISX0213_2000_ENCODE_BMP(assi, c) \ #define EMULATE_JISX0213_2000_ENCODE_BMP(config, assi, c) \
if (config == (void *)2000 && ( \ if ((config) == (void *)2000 && ( \
(c) == 0x9B1C || (c) == 0x4FF1 || \ (c) == 0x9B1C || (c) == 0x4FF1 || \
(c) == 0x525D || (c) == 0x541E || \ (c) == 0x525D || (c) == 0x541E || \
(c) == 0x5653 || (c) == 0x59F8 || \ (c) == 0x5653 || (c) == 0x59F8 || \
@ -14,12 +14,12 @@
(c) == 0x7626 || (c) == 0x7E6B)) { \ (c) == 0x7626 || (c) == 0x7E6B)) { \
return EMULATE_JISX0213_2000_ENCODE_INVALID; \ return EMULATE_JISX0213_2000_ENCODE_INVALID; \
} \ } \
else if (config == (void *)2000 && (c) == 0x9B1D) { \ else if ((config) == (void *)2000 && (c) == 0x9B1D) { \
(assi) = 0x8000 | 0x7d3b; \ (assi) = 0x8000 | 0x7d3b; \
} }
#define EMULATE_JISX0213_2000_ENCODE_EMP(assi, c) \ #define EMULATE_JISX0213_2000_ENCODE_EMP(config, assi, c) \
if (config == (void *)2000 && (c) == 0x20B9F) { \ if ((config) == (void *)2000 && (c) == 0x20B9F) { \
return EMULATE_JISX0213_2000_ENCODE_INVALID; \ return EMULATE_JISX0213_2000_ENCODE_INVALID; \
} }
@ -27,8 +27,8 @@
# define EMULATE_JISX0213_2000_DECODE_INVALID 2 # define EMULATE_JISX0213_2000_DECODE_INVALID 2
#endif #endif
#define EMULATE_JISX0213_2000_DECODE_PLANE1(assi, c1, c2) \ #define EMULATE_JISX0213_2000_DECODE_PLANE1(config, assi, c1, c2) \
if (config == (void *)2000 && \ if ((config) == (void *)2000 && \
(((c1) == 0x2E && (c2) == 0x21) || \ (((c1) == 0x2E && (c2) == 0x21) || \
((c1) == 0x2F && (c2) == 0x7E) || \ ((c1) == 0x2F && (c2) == 0x7E) || \
((c1) == 0x4F && (c2) == 0x54) || \ ((c1) == 0x4F && (c2) == 0x54) || \
@ -42,13 +42,13 @@
return EMULATE_JISX0213_2000_DECODE_INVALID; \ return EMULATE_JISX0213_2000_DECODE_INVALID; \
} }
#define EMULATE_JISX0213_2000_DECODE_PLANE2(writer, c1, c2) \ #define EMULATE_JISX0213_2000_DECODE_PLANE2(config, writer, c1, c2) \
if (config == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \ if ((config) == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
OUTCHAR(0x9B1D); \ OUTCHAR(0x9B1D); \
} }
#define EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(assi, c1, c2) \ #define EMULATE_JISX0213_2000_DECODE_PLANE2_CHAR(config, assi, c1, c2) \
if (config == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \ if ((config) == (void *)2000 && (c1) == 0x7D && (c2) == 0x3B) { \
(assi) = 0x9B1D; \ (assi) = 0x9B1D; \
} }

30
Modules/cjkcodecs/multibytecodec.c
View File

@ -272,7 +272,7 @@ multibytecodec_encerror(const MultibyteCodec *codec,
for (;;) { for (;;) {
Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf); Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
r = codec->encode(state, codec->config, r = codec->encode(state, codec,
kind, data, &inpos, 1, kind, data, &inpos, 1,
&buf->outbuf, outleft, 0); &buf->outbuf, outleft, 0);
if (r == MBERR_TOOSMALL) { if (r == MBERR_TOOSMALL) {
@ -521,7 +521,7 @@ multibytecodec_encode(const MultibyteCodec *codec,
* error callbacks can relocate the cursor anywhere on buffer*/ * error callbacks can relocate the cursor anywhere on buffer*/
Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
r = codec->encode(state, codec->config, r = codec->encode(state, codec,
kind, data, kind, data,
&buf.inpos, buf.inlen, &buf.inpos, buf.inlen,
&buf.outbuf, outleft, flags); &buf.outbuf, outleft, flags);
@ -538,7 +538,7 @@ multibytecodec_encode(const MultibyteCodec *codec,
Py_ssize_t outleft; Py_ssize_t outleft;
outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf); outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
r = codec->encreset(state, codec->config, &buf.outbuf, r = codec->encreset(state, codec, &buf.outbuf,
outleft); outleft);
if (r == 0) if (r == 0)
break; break;
@ -616,7 +616,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self,
} }
if (self->codec->encinit != NULL && if (self->codec->encinit != NULL &&
self->codec->encinit(&state, self->codec->config) != 0) self->codec->encinit(&state, self->codec) != 0)
goto errorexit; goto errorexit;
r = multibytecodec_encode(self->codec, &state, r = multibytecodec_encode(self->codec, &state,
input, NULL, errorcb, input, NULL, errorcb,
@ -680,7 +680,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
buf.inbuf_end = buf.inbuf_top + datalen; buf.inbuf_end = buf.inbuf_top + datalen;
if (self->codec->decinit != NULL && if (self->codec->decinit != NULL &&
self->codec->decinit(&state, self->codec->config) != 0) self->codec->decinit(&state, self->codec) != 0)
goto errorexit; goto errorexit;
while (buf.inbuf < buf.inbuf_end) { while (buf.inbuf < buf.inbuf_end) {
@ -688,7 +688,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self,
inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf); inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
r = self->codec->decode(&state, self->codec->config, r = self->codec->decode(&state, self->codec,
&buf.inbuf, inleft, &buf.writer); &buf.inbuf, inleft, &buf.writer);
if (r == 0) if (r == 0)
break; break;
@ -888,7 +888,7 @@ decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf); inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
r = ctx->codec->decode(&ctx->state, ctx->codec->config, r = ctx->codec->decode(&ctx->state, ctx->codec,
&buf->inbuf, inleft, &buf->writer); &buf->inbuf, inleft, &buf->writer);
if (r == 0 || r == MBERR_TOOFEW) if (r == 0 || r == MBERR_TOOFEW)
break; break;
@ -1015,7 +1015,7 @@ _multibytecodec_MultibyteIncrementalEncoder_reset_impl(MultibyteIncrementalEncod
Py_ssize_t r; Py_ssize_t r;
if (self->codec->encreset != NULL) { if (self->codec->encreset != NULL) {
outbuf = buffer; outbuf = buffer;
r = self->codec->encreset(&self->state, self->codec->config, r = self->codec->encreset(&self->state, self->codec,
&outbuf, sizeof(buffer)); &outbuf, sizeof(buffer));
if (r != 0) if (r != 0)
return NULL; return NULL;
@ -1063,7 +1063,7 @@ mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL) if (self->errors == NULL)
goto errorexit; goto errorexit;
if (self->codec->encinit != NULL && if (self->codec->encinit != NULL &&
self->codec->encinit(&self->state, self->codec->config) != 0) self->codec->encinit(&self->state, self->codec) != 0)
goto errorexit; goto errorexit;
Py_DECREF(codec); Py_DECREF(codec);
@ -1292,7 +1292,7 @@ _multibytecodec_MultibyteIncrementalDecoder_reset_impl(MultibyteIncrementalDecod
/*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/ /*[clinic end generated code: output=da423b1782c23ed1 input=3b63b3be85b2fb45]*/
{ {
if (self->codec->decreset != NULL && if (self->codec->decreset != NULL &&
self->codec->decreset(&self->state, self->codec->config) != 0) self->codec->decreset(&self->state, self->codec) != 0)
return NULL; return NULL;
self->pendingsize = 0; self->pendingsize = 0;
@ -1338,7 +1338,7 @@ mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL) if (self->errors == NULL)
goto errorexit; goto errorexit;
if (self->codec->decinit != NULL && if (self->codec->decinit != NULL &&
self->codec->decinit(&self->state, self->codec->config) != 0) self->codec->decinit(&self->state, self->codec) != 0)
goto errorexit; goto errorexit;
Py_DECREF(codec); Py_DECREF(codec);
@ -1600,7 +1600,7 @@ _multibytecodec_MultibyteStreamReader_reset_impl(MultibyteStreamReaderObject *se
/*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/ /*[clinic end generated code: output=138490370a680abc input=5d4140db84b5e1e2]*/
{ {
if (self->codec->decreset != NULL && if (self->codec->decreset != NULL &&
self->codec->decreset(&self->state, self->codec->config) != 0) self->codec->decreset(&self->state, self->codec) != 0)
return NULL; return NULL;
self->pendingsize = 0; self->pendingsize = 0;
@ -1654,7 +1654,7 @@ mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL) if (self->errors == NULL)
goto errorexit; goto errorexit;
if (self->codec->decinit != NULL && if (self->codec->decinit != NULL &&
self->codec->decinit(&self->state, self->codec->config) != 0) self->codec->decinit(&self->state, self->codec) != 0)
goto errorexit; goto errorexit;
Py_DECREF(codec); Py_DECREF(codec);
@ -1877,7 +1877,7 @@ mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (self->errors == NULL) if (self->errors == NULL)
goto errorexit; goto errorexit;
if (self->codec->encinit != NULL && if (self->codec->encinit != NULL &&
self->codec->encinit(&self->state, self->codec->config) != 0) self->codec->encinit(&self->state, self->codec) != 0)
goto errorexit; goto errorexit;
Py_DECREF(codec); Py_DECREF(codec);
@ -1971,7 +1971,7 @@ _multibytecodec___create_codec(PyObject *module, PyObject *arg)
codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE); codec_capsule *data = PyCapsule_GetPointer(arg, CODEC_CAPSULE);
const MultibyteCodec *codec = data->codec; const MultibyteCodec *codec = data->codec;
if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0) if (codec->codecinit != NULL && codec->codecinit(codec) != 0)
return NULL; return NULL;
module_state *state = get_module_state(module); module_state *state = get_module_state(module);

20
Modules/cjkcodecs/multibytecodec.h
View File

@ -27,28 +27,31 @@ typedef struct {
unsigned char c[8]; unsigned char c[8];
} MultibyteCodec_State; } MultibyteCodec_State;
typedef int (*mbcodec_init)(const void *config); struct _cjk_mod_state;
struct _multibyte_codec;
typedef int (*mbcodec_init)(const struct _multibyte_codec *codec);
typedef Py_ssize_t (*mbencode_func)(MultibyteCodec_State *state, typedef Py_ssize_t (*mbencode_func)(MultibyteCodec_State *state,
const void *config, const struct _multibyte_codec *codec,
int kind, const void *data, int kind, const void *data,
Py_ssize_t *inpos, Py_ssize_t inlen, Py_ssize_t *inpos, Py_ssize_t inlen,
unsigned char **outbuf, Py_ssize_t outleft, unsigned char **outbuf, Py_ssize_t outleft,
int flags); int flags);
typedef int (*mbencodeinit_func)(MultibyteCodec_State *state, typedef int (*mbencodeinit_func)(MultibyteCodec_State *state,
const void *config); const struct _multibyte_codec *codec);
typedef Py_ssize_t (*mbencodereset_func)(MultibyteCodec_State *state, typedef Py_ssize_t (*mbencodereset_func)(MultibyteCodec_State *state,
const void *config, const struct _multibyte_codec *codec,
unsigned char **outbuf, Py_ssize_t outleft); unsigned char **outbuf, Py_ssize_t outleft);
typedef Py_ssize_t (*mbdecode_func)(MultibyteCodec_State *state, typedef Py_ssize_t (*mbdecode_func)(MultibyteCodec_State *state,
const void *config, const struct _multibyte_codec *codec,
const unsigned char **inbuf, Py_ssize_t inleft, const unsigned char **inbuf, Py_ssize_t inleft,
_PyUnicodeWriter *writer); _PyUnicodeWriter *writer);
typedef int (*mbdecodeinit_func)(MultibyteCodec_State *state, typedef int (*mbdecodeinit_func)(MultibyteCodec_State *state,
const void *config); const struct _multibyte_codec *codec);
typedef Py_ssize_t (*mbdecodereset_func)(MultibyteCodec_State *state, typedef Py_ssize_t (*mbdecodereset_func)(MultibyteCodec_State *state,
const void *config); const struct _multibyte_codec *codec);
typedef struct { typedef struct _multibyte_codec {
const char *encoding; const char *encoding;
const void *config; const void *config;
mbcodec_init codecinit; mbcodec_init codecinit;
@ -58,6 +61,7 @@ typedef struct {
mbdecode_func decode; mbdecode_func decode;
mbdecodeinit_func decinit; mbdecodeinit_func decinit;
mbdecodereset_func decreset; mbdecodereset_func decreset;
struct _cjk_mod_state *modstate;
} MultibyteCodec; } MultibyteCodec;
typedef struct { typedef struct {

21
Tools/c-analyzer/cpython/globals-to-fix.tsv
View File

@ -481,27 +481,6 @@ Modules/_decimal/_decimal.c - _py_float_abs -
Modules/_decimal/_decimal.c - _py_long_bit_length - Modules/_decimal/_decimal.c - _py_long_bit_length -
Modules/_decimal/_decimal.c - _py_float_as_integer_ratio - Modules/_decimal/_decimal.c - _py_float_as_integer_ratio -
Modules/_elementtree.c - expat_capi - Modules/_elementtree.c - expat_capi -
Modules/cjkcodecs/_codecs_hk.c - big5_encmap -
Modules/cjkcodecs/_codecs_hk.c - big5_decmap -
Modules/cjkcodecs/_codecs_hk.c big5hkscs_codec_init initialized -
Modules/cjkcodecs/_codecs_iso2022.c - cp949_encmap -
Modules/cjkcodecs/_codecs_iso2022.c - ksx1001_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisxcommon_encmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0208_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0212_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_bmp_encmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_1_bmp_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_2_bmp_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_emp_encmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_1_emp_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - jisx0213_2_emp_decmap -
Modules/cjkcodecs/_codecs_iso2022.c - gbcommon_encmap -
Modules/cjkcodecs/_codecs_iso2022.c - gb2312_decmap -
Modules/cjkcodecs/_codecs_iso2022.c ksx1001_init initialized -
Modules/cjkcodecs/_codecs_iso2022.c jisx0208_init initialized -
Modules/cjkcodecs/_codecs_iso2022.c jisx0212_init initialized -
Modules/cjkcodecs/_codecs_iso2022.c jisx0213_init initialized -
Modules/cjkcodecs/_codecs_iso2022.c gb2312_init initialized -
Modules/readline.c - libedit_append_replace_history_offset - Modules/readline.c - libedit_append_replace_history_offset -
Modules/readline.c - using_libedit_emulation - Modules/readline.c - using_libedit_emulation -
Modules/readline.c - libedit_history_start - Modules/readline.c - libedit_history_start -

Can't render this file because it has a wrong number of fields in line 4.