traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
cpython/Modules/unicodedata.c

1383 lines
41 KiB
C
Raw Blame History

/* ------------------------------------------------------------------------
unicodedata -- Provides access to the Unicode database.
Data was extracted from the UnicodeData.txt file.
The current version number is reported in the unidata_version constant.
Written by Marc-Andre Lemburg (mal@lemburg.com).
Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
Modified by Martin v. Löwis (martin@v.loewis.de)
Copyright (c) Corporation for National Research Initiatives.
------------------------------------------------------------------------ */
#include "Python.h"
#include "ucnhash.h"
#include "structmember.h"
/* character properties */
typedef struct {
const unsigned char category; /* index into
_PyUnicode_CategoryNames */
const unsigned char combining; /* combining class value 0 - 255 */
const unsigned char bidirectional; /* index into
_PyUnicode_BidirectionalNames */
const unsigned char mirrored; /* true if mirrored in bidir mode */
const unsigned char east_asian_width; /* index into
_PyUnicode_EastAsianWidth */
const unsigned char normalization_quick_check; /* see is_normalized() */
} _PyUnicode_DatabaseRecord;
typedef struct change_record {
/* sequence of fields should be the same as in merge_old_version */
const unsigned char bidir_changed;
const unsigned char category_changed;
const unsigned char decimal_changed;
const unsigned char mirrored_changed;
const double numeric_changed;
} change_record;
/* data file generated by Tools/unicode/makeunicodedata.py */
#include "unicodedata_db.h"
static const _PyUnicode_DatabaseRecord*
_getrecord_ex(Py_UCS4 code)
{
int index;
if (code >= 0x110000)
index = 0;
else {
index = index1[(code>>SHIFT)];
index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
}
return &_PyUnicode_Database_Records[index];
}
/* ------------- Previous-version API ------------------------------------- */
typedef struct previous_version {
PyObject_HEAD
const char *name;
const change_record* (*getrecord)(Py_UCS4);
Py_UCS4 (*normalization)(Py_UCS4);
} PreviousDBVersion;
#define get_old_record(self, v) ((((PreviousDBVersion*)self)->getrecord)(v))
static PyMemberDef DB_members[] = {
{"unidata_version", T_STRING, offsetof(PreviousDBVersion, name), READONLY},
{NULL}
};
/* forward declaration */
static PyTypeObject UCD_Type;
#define UCD_Check(o) (Py_TYPE(o)==&UCD_Type)
static PyObject*
new_previous_version(const char*name, const change_record* (*getrecord)(Py_UCS4),
Py_UCS4 (*normalization)(Py_UCS4))
{
PreviousDBVersion *self;
self = PyObject_New(PreviousDBVersion, &UCD_Type);
if (self == NULL)
return NULL;
self->name = name;
self->getrecord = getrecord;
self->normalization = normalization;
return (PyObject*)self;
}
static Py_UCS4 getuchar(PyUnicodeObject *obj)
{
if (PyUnicode_READY(obj))
return (Py_UCS4)-1;
if (PyUnicode_GET_LENGTH(obj) == 1) {
if (PyUnicode_READY(obj))
return (Py_UCS4)-1;
return PyUnicode_READ_CHAR(obj, 0);
}
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
return (Py_UCS4)-1;
}
/* --- Module API --------------------------------------------------------- */
PyDoc_STRVAR(unicodedata_decimal__doc__,
"decimal(unichr[, default])\n\
\n\
Returns the decimal value assigned to the Unicode character unichr\n\
as integer. If no such value is defined, default is returned, or, if\n\
not given, ValueError is raised.");
static PyObject *
unicodedata_decimal(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
int have_old = 0;
long rc;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!|O:decimal", &PyUnicode_Type, &v, &defobj))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0) {
/* unassigned */
have_old = 1;
rc = -1;
}
else if (old->decimal_changed != 0xFF) {
have_old = 1;
rc = old->decimal_changed;
}
}
if (!have_old)
rc = Py_UNICODE_TODECIMAL(c);
if (rc < 0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError,
"not a decimal");
return NULL;
}
else {
Py_INCREF(defobj);
return defobj;
}
}
return PyLong_FromLong(rc);
}
PyDoc_STRVAR(unicodedata_digit__doc__,
"digit(unichr[, default])\n\
\n\
Returns the digit value assigned to the Unicode character unichr as\n\
integer. If no such value is defined, default is returned, or, if\n\
not given, ValueError is raised.");
static PyObject *
unicodedata_digit(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
long rc;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!|O:digit", &PyUnicode_Type, &v, &defobj))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
rc = Py_UNICODE_TODIGIT(c);
if (rc < 0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError, "not a digit");
return NULL;
}
else {
Py_INCREF(defobj);
return defobj;
}
}
return PyLong_FromLong(rc);
}
PyDoc_STRVAR(unicodedata_numeric__doc__,
"numeric(unichr[, default])\n\
\n\
Returns the numeric value assigned to the Unicode character unichr\n\
as float. If no such value is defined, default is returned, or, if\n\
not given, ValueError is raised.");
static PyObject *
unicodedata_numeric(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
int have_old = 0;
double rc;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!|O:numeric", &PyUnicode_Type, &v, &defobj))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0) {
/* unassigned */
have_old = 1;
rc = -1.0;
}
else if (old->decimal_changed != 0xFF) {
have_old = 1;
rc = old->decimal_changed;
}
}
if (!have_old)
rc = Py_UNICODE_TONUMERIC(c);
if (rc == -1.0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError, "not a numeric character");
return NULL;
}
else {
Py_INCREF(defobj);
return defobj;
}
}
return PyFloat_FromDouble(rc);
}
PyDoc_STRVAR(unicodedata_category__doc__,
"category(unichr)\n\
\n\
Returns the general category assigned to the Unicode character\n\
unichr as string.");
static PyObject *
unicodedata_category(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:category",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
index = (int) _getrecord_ex(c)->category;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed != 0xFF)
index = old->category_changed;
}
return PyUnicode_FromString(_PyUnicode_CategoryNames[index]);
}
PyDoc_STRVAR(unicodedata_bidirectional__doc__,
"bidirectional(unichr)\n\
\n\
Returns the bidirectional class assigned to the Unicode character\n\
unichr as string. If no such value is defined, an empty string is\n\
returned.");
static PyObject *
unicodedata_bidirectional(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:bidirectional",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
index = (int) _getrecord_ex(c)->bidirectional;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0)
index = 0; /* unassigned */
else if (old->bidir_changed != 0xFF)
index = old->bidir_changed;
}
return PyUnicode_FromString(_PyUnicode_BidirectionalNames[index]);
}
PyDoc_STRVAR(unicodedata_combining__doc__,
"combining(unichr)\n\
\n\
Returns the canonical combining class assigned to the Unicode\n\
character unichr as integer. Returns 0 if no combining class is\n\
defined.");
static PyObject *
unicodedata_combining(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:combining",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
index = (int) _getrecord_ex(c)->combining;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0)
index = 0; /* unassigned */
}
return PyLong_FromLong(index);
}
PyDoc_STRVAR(unicodedata_mirrored__doc__,
"mirrored(unichr)\n\
\n\
Returns the mirrored property assigned to the Unicode character\n\
unichr as integer. Returns 1 if the character has been identified as\n\
a \"mirrored\" character in bidirectional text, 0 otherwise.");
static PyObject *
unicodedata_mirrored(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:mirrored",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
index = (int) _getrecord_ex(c)->mirrored;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0)
index = 0; /* unassigned */
else if (old->mirrored_changed != 0xFF)
index = old->mirrored_changed;
}
return PyLong_FromLong(index);
}
PyDoc_STRVAR(unicodedata_east_asian_width__doc__,
"east_asian_width(unichr)\n\
\n\
Returns the east asian width assigned to the Unicode character\n\
unichr as string.");
static PyObject *
unicodedata_east_asian_width(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:east_asian_width",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
index = (int) _getrecord_ex(c)->east_asian_width;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0)
index = 0; /* unassigned */
}
return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]);
}
PyDoc_STRVAR(unicodedata_decomposition__doc__,
"decomposition(unichr)\n\
\n\
Returns the character decomposition mapping assigned to the Unicode\n\
character unichr as string. An empty string is returned in case no\n\
such mapping is defined.");
static PyObject *
unicodedata_decomposition(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
char decomp[256];
int code, index, count;
size_t i;
unsigned int prefix_index;
Py_UCS4 c;
if (!PyArg_ParseTuple(args, "O!:decomposition",
&PyUnicode_Type, &v))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
code = (int)c;
if (self && UCD_Check(self)) {
const change_record *old = get_old_record(self, c);
if (old->category_changed == 0)
return PyUnicode_FromString(""); /* unassigned */
}
if (code < 0 || code >= 0x110000)
index = 0;
else {
index = decomp_index1[(code>>DECOMP_SHIFT)];
index = decomp_index2[(index<<DECOMP_SHIFT)+
(code&((1<<DECOMP_SHIFT)-1))];
}
/* high byte is number of hex bytes (usually one or two), low byte
is prefix code (from*/
count = decomp_data[index] >> 8;
/* XXX: could allocate the PyString up front instead
(strlen(prefix) + 5 * count + 1 bytes) */
/* Based on how index is calculated above and decomp_data is generated
from Tools/unicode/makeunicodedata.py, it should not be possible
to overflow decomp_prefix. */
prefix_index = decomp_data[index] & 255;
assert(prefix_index < Py_ARRAY_LENGTH(decomp_prefix));
/* copy prefix */
i = strlen(decomp_prefix[prefix_index]);
memcpy(decomp, decomp_prefix[prefix_index], i);
while (count-- > 0) {
if (i)
decomp[i++] = ' ';
assert(i < sizeof(decomp));
PyOS_snprintf(decomp + i, sizeof(decomp) - i, "%04X",
decomp_data[++index]);
i += strlen(decomp + i);
}
return PyUnicode_FromStringAndSize(decomp, i);
}
static void
get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *count)
{
if (code >= 0x110000) {
*index = 0;
} else if (self && UCD_Check(self) &&
get_old_record(self, code)->category_changed==0) {
/* unassigned in old version */
*index = 0;
}
else {
*index = decomp_index1[(code>>DECOMP_SHIFT)];
*index = decomp_index2[(*index<<DECOMP_SHIFT)+
(code&((1<<DECOMP_SHIFT)-1))];
}
/* high byte is number of hex bytes (usually one or two), low byte
is prefix code (from*/
*count = decomp_data[*index] >> 8;
*prefix = decomp_data[*index] & 255;
(*index)++;
}
#define SBase 0xAC00
#define LBase 0x1100
#define VBase 0x1161
#define TBase 0x11A7
#define LCount 19
#define VCount 21
#define TCount 28
#define NCount (VCount*TCount)
#define SCount (LCount*NCount)
static PyObject*
nfd_nfkd(PyObject *self, PyObject *input, int k)
{
PyObject *result;
Py_UCS4 *output;
Py_ssize_t i, o, osize;
int kind;
void *data;
/* Longest decomposition in Unicode 3.2: U+FDFA */
Py_UCS4 stack[20];
Py_ssize_t space, isize;
int index, prefix, count, stackptr;
unsigned char prev, cur;
stackptr = 0;
isize = PyUnicode_GET_LENGTH(input);
space = isize;
/* Overallocate at most 10 characters. */
if (space > 10) {
if (space <= PY_SSIZE_T_MAX - 10)
space += 10;
}
else {
space *= 2;
}
osize = space;
output = PyMem_NEW(Py_UCS4, space);
if (!output) {
PyErr_NoMemory();
return NULL;
}
i = o = 0;
kind = PyUnicode_KIND(input);
data = PyUnicode_DATA(input);
while (i < isize) {
stack[stackptr++] = PyUnicode_READ(kind, data, i++);
while(stackptr) {
Py_UCS4 code = stack[--stackptr];
/* Hangul Decomposition adds three characters in
a single step, so we need at least that much room. */
if (space < 3) {
Py_UCS4 *new_output;
osize += 10;
space += 10;
new_output = PyMem_Realloc(output, osize*sizeof(Py_UCS4));
if (new_output == NULL) {
PyMem_Free(output);
PyErr_NoMemory();
return NULL;
}
output = new_output;
}
/* Hangul Decomposition. */
if (SBase <= code && code < (SBase+SCount)) {
int SIndex = code - SBase;
int L = LBase + SIndex / NCount;
int V = VBase + (SIndex % NCount) / TCount;
int T = TBase + SIndex % TCount;
output[o++] = L;
output[o++] = V;
space -= 2;
if (T != TBase) {
output[o++] = T;
space --;
}
continue;
}
/* normalization changes */
if (self && UCD_Check(self)) {
Py_UCS4 value = ((PreviousDBVersion*)self)->normalization(code);
if (value != 0) {
stack[stackptr++] = value;
continue;
}
}
/* Other decompositions. */
get_decomp_record(self, code, &index, &prefix, &count);
/* Copy character if it is not decomposable, or has a
compatibility decomposition, but we do NFD. */
if (!count || (prefix && !k)) {
output[o++] = code;
space--;
continue;
}
/* Copy decomposition onto the stack, in reverse
order. */
while(count) {
code = decomp_data[index + (--count)];
stack[stackptr++] = code;
}
}
}
result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
output, o);
PyMem_Free(output);
if (!result)
return NULL;
/* result is guaranteed to be ready, as it is compact. */
kind = PyUnicode_KIND(result);
data = PyUnicode_DATA(result);
/* Sort canonically. */
i = 0;
prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
for (i++; i < PyUnicode_GET_LENGTH(result); i++) {
cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
if (prev == 0 || cur == 0 || prev <= cur) {
prev = cur;
continue;
}
/* Non-canonical order. Need to switch *i with previous. */
o = i - 1;
while (1) {
Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1);
PyUnicode_WRITE(kind, data, o+1,
PyUnicode_READ(kind, data, o));
PyUnicode_WRITE(kind, data, o, tmp);
o--;
if (o < 0)
break;
prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining;
if (prev == 0 || prev <= cur)
break;
}
prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
}
return result;
}
static int
find_nfc_index(PyObject *self, struct reindex* nfc, Py_UCS4 code)
{
unsigned int index;
for (index = 0; nfc[index].start; index++) {
unsigned int start = nfc[index].start;
if (code < start)
return -1;
if (code <= start + nfc[index].count) {
unsigned int delta = code - start;
return nfc[index].index + delta;
}
}
return -1;
}
static PyObject*
nfc_nfkc(PyObject *self, PyObject *input, int k)
{
PyObject *result;
int kind;
void *data;
Py_UCS4 *output;
Py_ssize_t i, i1, o, len;
int f,l,index,index1,comb;
Py_UCS4 code;
Py_ssize_t skipped[20];
int cskipped = 0;
result = nfd_nfkd(self, input, k);
if (!result)
return NULL;
/* result will be "ready". */
kind = PyUnicode_KIND(result);
data = PyUnicode_DATA(result);
len = PyUnicode_GET_LENGTH(result);
/* We allocate a buffer for the output.
If we find that we made no changes, we still return
the NFD result. */
output = PyMem_NEW(Py_UCS4, len);
if (!output) {
PyErr_NoMemory();
Py_DECREF(result);
return 0;
}
i = o = 0;
again:
while (i < len) {
for (index = 0; index < cskipped; index++) {
if (skipped[index] == i) {
/* *i character is skipped.
Remove from list. */
skipped[index] = skipped[cskipped-1];
cskipped--;
i++;
goto again; /* continue while */
}
}
/* Hangul Composition. We don't need to check for <LV,T>
pairs, since we always have decomposed data. */
code = PyUnicode_READ(kind, data, i);
if (LBase <= code && code < (LBase+LCount) &&
i + 1 < len &&
VBase <= PyUnicode_READ(kind, data, i+1) &&
PyUnicode_READ(kind, data, i+1) <= (VBase+VCount)) {
int LIndex, VIndex;
LIndex = code - LBase;
VIndex = PyUnicode_READ(kind, data, i+1) - VBase;
code = SBase + (LIndex*VCount+VIndex)*TCount;
i+=2;
if (i < len &&
TBase <= PyUnicode_READ(kind, data, i) &&
PyUnicode_READ(kind, data, i) <= (TBase+TCount)) {
code += PyUnicode_READ(kind, data, i)-TBase;
i++;
}
output[o++] = code;
continue;
}
/* code is still input[i] here */
f = find_nfc_index(self, nfc_first, code);
if (f == -1) {
output[o++] = code;
i++;
continue;
}
/* Find next unblocked character. */
i1 = i+1;
comb = 0;
/* output base character for now; might be updated later. */
output[o] = PyUnicode_READ(kind, data, i);
while (i1 < len) {
Py_UCS4 code1 = PyUnicode_READ(kind, data, i1);
int comb1 = _getrecord_ex(code1)->combining;
if (comb) {
if (comb1 == 0)
break;
if (comb >= comb1) {
/* Character is blocked. */
i1++;
continue;
}
}
l = find_nfc_index(self, nfc_last, code1);
/* i1 cannot be combined with i. If i1
is a starter, we don't need to look further.
Otherwise, record the combining class. */
if (l == -1) {
not_combinable:
if (comb1 == 0)
break;
comb = comb1;
i1++;
continue;
}
index = f*TOTAL_LAST + l;
index1 = comp_index[index >> COMP_SHIFT];
code = comp_data[(index1<<COMP_SHIFT)+
(index&((1<<COMP_SHIFT)-1))];
if (code == 0)
goto not_combinable;
/* Replace the original character. */
output[o] = code;
/* Mark the second character unused. */
assert(cskipped < 20);
skipped[cskipped++] = i1;
i1++;
f = find_nfc_index(self, nfc_first, output[o]);
if (f == -1)
break;
}
/* Output character was already written.
Just advance the indices. */
o++; i++;
}
if (o == len) {
/* No changes. Return original string. */
PyMem_Free(output);
return result;
}
Py_DECREF(result);
result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
output, o);
PyMem_Free(output);
return result;
}
/* Return 1 if the input is certainly normalized, 0 if it might not be. */
static int
is_normalized(PyObject *self, PyObject *input, int nfc, int k)
{
Py_ssize_t i, len;
int kind;
void *data;
unsigned char prev_combining = 0, quickcheck_mask;
/* An older version of the database is requested, quickchecks must be
disabled. */
if (self && UCD_Check(self))
return 0;
/* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
as described in http://unicode.org/reports/tr15/#Annex8. */
quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
i = 0;
kind = PyUnicode_KIND(input);
data = PyUnicode_DATA(input);
len = PyUnicode_GET_LENGTH(input);
while (i < len) {
Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
unsigned char combining = record->combining;
unsigned char quickcheck = record->normalization_quick_check;
if (quickcheck & quickcheck_mask)
return 0; /* this string might need normalization */
if (combining && prev_combining > combining)
return 0; /* non-canonical sort order, not normalized */
prev_combining = combining;
}
return 1; /* certainly normalized */
}
PyDoc_STRVAR(unicodedata_normalize__doc__,
"normalize(form, unistr)\n\
\n\
Return the normal form 'form' for the Unicode string unistr. Valid\n\
values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.");
static PyObject*
unicodedata_normalize(PyObject *self, PyObject *args)
{
char *form;
PyObject *input;
if(!PyArg_ParseTuple(args, "sO!:normalize",
&form, &PyUnicode_Type, &input))
return NULL;
if (PyUnicode_READY(input) == -1)
return NULL;
if (PyUnicode_GET_LENGTH(input) == 0) {
/* Special case empty input strings, since resizing
them later would cause internal errors. */
Py_INCREF(input);
return input;
}
if (strcmp(form, "NFC") == 0) {
if (is_normalized(self, input, 1, 0)) {
Py_INCREF(input);
return input;
}
return nfc_nfkc(self, input, 0);
}
if (strcmp(form, "NFKC") == 0) {
if (is_normalized(self, input, 1, 1)) {
Py_INCREF(input);
return input;
}
return nfc_nfkc(self, input, 1);
}
if (strcmp(form, "NFD") == 0) {
if (is_normalized(self, input, 0, 0)) {
Py_INCREF(input);
return input;
}
return nfd_nfkd(self, input, 0);
}
if (strcmp(form, "NFKD") == 0) {
if (is_normalized(self, input, 0, 1)) {
Py_INCREF(input);
return input;
}
return nfd_nfkd(self, input, 1);
}
PyErr_SetString(PyExc_ValueError, "invalid normalization form");
return NULL;
}
/* -------------------------------------------------------------------- */
/* unicode character name tables */
/* data file generated by Tools/unicode/makeunicodedata.py */
#include "unicodename_db.h"
/* -------------------------------------------------------------------- */
/* database code (cut and pasted from the unidb package) */
static unsigned long
_gethash(const char *s, int len, int scale)
{
int i;
unsigned long h = 0;
unsigned long ix;
for (i = 0; i < len; i++) {
h = (h * scale) + (unsigned char) Py_TOUPPER(Py_CHARMASK(s[i]));
ix = h & 0xff000000;
if (ix)
h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
}
return h;
}
static char *hangul_syllables[][3] = {
{ "G", "A", "" },
{ "GG", "AE", "G" },
{ "N", "YA", "GG" },
{ "D", "YAE", "GS" },
{ "DD", "EO", "N", },
{ "R", "E", "NJ" },
{ "M", "YEO", "NH" },
{ "B", "YE", "D" },
{ "BB", "O", "L" },
{ "S", "WA", "LG" },
{ "SS", "WAE", "LM" },
{ "", "OE", "LB" },
{ "J", "YO", "LS" },
{ "JJ", "U", "LT" },
{ "C", "WEO", "LP" },
{ "K", "WE", "LH" },
{ "T", "WI", "M" },
{ "P", "YU", "B" },
{ "H", "EU", "BS" },
{ 0, "YI", "S" },
{ 0, "I", "SS" },
{ 0, 0, "NG" },
{ 0, 0, "J" },
{ 0, 0, "C" },
{ 0, 0, "K" },
{ 0, 0, "T" },
{ 0, 0, "P" },
{ 0, 0, "H" }
};
/* These ranges need to match makeunicodedata.py:cjk_ranges. */
static int
is_unified_ideograph(Py_UCS4 code)
{
return
(0x3400 <= code && code <= 0x4DB5) || /* CJK Ideograph Extension A */
(0x4E00 <= code && code <= 0x9FCC) || /* CJK Ideograph */
(0x20000 <= code && code <= 0x2A6D6) || /* CJK Ideograph Extension B */
(0x2A700 <= code && code <= 0x2B734) || /* CJK Ideograph Extension C */
(0x2B740 <= code && code <= 0x2B81D); /* CJK Ideograph Extension D */
}
/* macros used to determine if the given codepoint is in the PUA range that
* we are using to store aliases and named sequences */
#define IS_ALIAS(cp) ((cp >= aliases_start) && (cp < aliases_end))
#define IS_NAMED_SEQ(cp) ((cp >= named_sequences_start) && \
(cp < named_sequences_end))
static int
_getucname(PyObject *self, Py_UCS4 code, char* buffer, int buflen,
int with_alias_and_seq)
{
/* Find the name associated with the given codepoint.
* If with_alias_and_seq is 1, check for names in the Private Use Area 15
* that we are using for aliases and named sequences. */
int offset;
int i;
int word;
unsigned char* w;
if (code >= 0x110000)
return 0;
/* XXX should we just skip all the codepoints in the PUAs here? */
if (!with_alias_and_seq && (IS_ALIAS(code) || IS_NAMED_SEQ(code)))
return 0;
if (self && UCD_Check(self)) {
/* in 3.2.0 there are no aliases and named sequences */
const change_record *old;
if (IS_ALIAS(code) || IS_NAMED_SEQ(code))
return 0;
old = get_old_record(self, code);
if (old->category_changed == 0) {
/* unassigned */
return 0;
}
}
if (SBase <= code && code < SBase+SCount) {
/* Hangul syllable. */
int SIndex = code - SBase;
int L = SIndex / NCount;
int V = (SIndex % NCount) / TCount;
int T = SIndex % TCount;
if (buflen < 27)
/* Worst case: HANGUL SYLLABLE <10chars>. */
return 0;
strcpy(buffer, "HANGUL SYLLABLE ");
buffer += 16;
strcpy(buffer, hangul_syllables[L][0]);
buffer += strlen(hangul_syllables[L][0]);
strcpy(buffer, hangul_syllables[V][1]);
buffer += strlen(hangul_syllables[V][1]);
strcpy(buffer, hangul_syllables[T][2]);
buffer += strlen(hangul_syllables[T][2]);
*buffer = '\0';
return 1;
}
if (is_unified_ideograph(code)) {
if (buflen < 28)
/* Worst case: CJK UNIFIED IDEOGRAPH-20000 */
return 0;
sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code);
return 1;
}
/* get offset into phrasebook */
offset = phrasebook_offset1[(code>>phrasebook_shift)];
offset = phrasebook_offset2[(offset<<phrasebook_shift) +
(code&((1<<phrasebook_shift)-1))];
if (!offset)
return 0;
i = 0;
for (;;) {
/* get word index */
word = phrasebook[offset] - phrasebook_short;
if (word >= 0) {
word = (word << 8) + phrasebook[offset+1];
offset += 2;
} else
word = phrasebook[offset++];
if (i) {
if (i > buflen)
return 0; /* buffer overflow */
buffer[i++] = ' ';
}
/* copy word string from lexicon. the last character in the
word has bit 7 set. the last word in a string ends with
0x80 */
w = lexicon + lexicon_offset[word];
while (*w < 128) {
if (i >= buflen)
return 0; /* buffer overflow */
buffer[i++] = *w++;
}
if (i >= buflen)
return 0; /* buffer overflow */
buffer[i++] = *w & 127;
if (*w == 128)
break; /* end of word */
}
return 1;
}
static int
_cmpname(PyObject *self, int code, const char* name, int namelen)
{
/* check if code corresponds to the given name */
int i;
char buffer[NAME_MAXLEN];
if (!_getucname(self, code, buffer, sizeof(buffer), 1))
return 0;
for (i = 0; i < namelen; i++) {
if (Py_TOUPPER(Py_CHARMASK(name[i])) != buffer[i])
return 0;
}
return buffer[namelen] == '\0';
}
static void
find_syllable(const char *str, int *len, int *pos, int count, int column)
{
int i, len1;
*len = -1;
for (i = 0; i < count; i++) {
char *s = hangul_syllables[i][column];
len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int);
if (len1 <= *len)
continue;
if (strncmp(str, s, len1) == 0) {
*len = len1;
*pos = i;
}
}
if (*len == -1) {
*len = 0;
}
}
static int
_check_alias_and_seq(unsigned int cp, Py_UCS4* code, int with_named_seq)
{
/* check if named sequences are allowed */
if (!with_named_seq && IS_NAMED_SEQ(cp))
return 0;
/* if the codepoint is in the PUA range that we use for aliases,
* convert it to obtain the right codepoint */
if (IS_ALIAS(cp))
*code = name_aliases[cp-aliases_start];
else
*code = cp;
return 1;
}
static int
_getcode(PyObject* self, const char* name, int namelen, Py_UCS4* code,
int with_named_seq)
{
/* Return the codepoint associated with the given name.
* Named aliases are resolved too (unless self != NULL (i.e. we are using
* 3.2.0)). If with_named_seq is 1, returns the PUA codepoint that we are
* using for the named sequence, and the caller must then convert it. */
unsigned int h, v;
unsigned int mask = code_size-1;
unsigned int i, incr;
/* Check for hangul syllables. */
if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) {
int len, L = -1, V = -1, T = -1;
const char *pos = name + 16;
find_syllable(pos, &len, &L, LCount, 0);
pos += len;
find_syllable(pos, &len, &V, VCount, 1);
pos += len;
find_syllable(pos, &len, &T, TCount, 2);
pos += len;
if (L != -1 && V != -1 && T != -1 && pos-name == namelen) {
*code = SBase + (L*VCount+V)*TCount + T;
return 1;
}
/* Otherwise, it's an illegal syllable name. */
return 0;
}
/* Check for unified ideographs. */
if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) {
/* Four or five hexdigits must follow. */
v = 0;
name += 22;
namelen -= 22;
if (namelen != 4 && namelen != 5)
return 0;
while (namelen--) {
v *= 16;
if (*name >= '0' && *name <= '9')
v += *name - '0';
else if (*name >= 'A' && *name <= 'F')
v += *name - 'A' + 10;
else
return 0;
name++;
}
if (!is_unified_ideograph(v))
return 0;
*code = v;
return 1;
}
/* the following is the same as python's dictionary lookup, with
only minor changes. see the makeunicodedata script for more
details */
h = (unsigned int) _gethash(name, namelen, code_magic);
i = (~h) & mask;
v = code_hash[i];
if (!v)
return 0;
if (_cmpname(self, v, name, namelen))
return _check_alias_and_seq(v, code, with_named_seq);
incr = (h ^ (h >> 3)) & mask;
if (!incr)
incr = mask;
for (;;) {
i = (i + incr) & mask;
v = code_hash[i];
if (!v)
return 0;
if (_cmpname(self, v, name, namelen))
return _check_alias_and_seq(v, code, with_named_seq);
incr = incr << 1;
if (incr > mask)
incr = incr ^ code_poly;
}
}
static const _PyUnicode_Name_CAPI hashAPI =
{
sizeof(_PyUnicode_Name_CAPI),
_getucname,
_getcode
};
/* -------------------------------------------------------------------- */
/* Python bindings */
PyDoc_STRVAR(unicodedata_name__doc__,
"name(unichr[, default])\n\
Returns the name assigned to the Unicode character unichr as a\n\
string. If no name is defined, default is returned, or, if not\n\
given, ValueError is raised.");
static PyObject *
unicodedata_name(PyObject* self, PyObject* args)
{
char name[NAME_MAXLEN];
Py_UCS4 c;
PyUnicodeObject* v;
PyObject* defobj = NULL;
if (!PyArg_ParseTuple(args, "O!|O:name", &PyUnicode_Type, &v, &defobj))
return NULL;
c = getuchar(v);
if (c == (Py_UCS4)-1)
return NULL;
if (!_getucname(self, c, name, sizeof(name), 0)) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError, "no such name");
return NULL;
}
else {
Py_INCREF(defobj);
return defobj;
}
}
return PyUnicode_FromString(name);
}
PyDoc_STRVAR(unicodedata_lookup__doc__,
"lookup(name)\n\
\n\
Look up character by name. If a character with the\n\
given name is found, return the corresponding Unicode\n\
character. If not found, KeyError is raised.");
static PyObject *
unicodedata_lookup(PyObject* self, PyObject* args)
{
Py_UCS4 code;
char* name;
int namelen;
unsigned int index;
if (!PyArg_ParseTuple(args, "s#:lookup", &name, &namelen))
return NULL;
if (!_getcode(self, name, namelen, &code, 1)) {
PyErr_Format(PyExc_KeyError, "undefined character name '%s'", name);
return NULL;
}
/* check if code is in the PUA range that we use for named sequences
and convert it */
if (IS_NAMED_SEQ(code)) {
index = code-named_sequences_start;
return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND,
named_sequences[index].seq,
named_sequences[index].seqlen);
}
return PyUnicode_FromOrdinal(code);
}
/* XXX Add doc strings. */
static PyMethodDef unicodedata_functions[] = {
{"decimal", unicodedata_decimal, METH_VARARGS, unicodedata_decimal__doc__},
{"digit", unicodedata_digit, METH_VARARGS, unicodedata_digit__doc__},
{"numeric", unicodedata_numeric, METH_VARARGS, unicodedata_numeric__doc__},
{"category", unicodedata_category, METH_VARARGS,
unicodedata_category__doc__},
{"bidirectional", unicodedata_bidirectional, METH_VARARGS,
unicodedata_bidirectional__doc__},
{"combining", unicodedata_combining, METH_VARARGS,
unicodedata_combining__doc__},
{"mirrored", unicodedata_mirrored, METH_VARARGS,
unicodedata_mirrored__doc__},
{"east_asian_width", unicodedata_east_asian_width, METH_VARARGS,
unicodedata_east_asian_width__doc__},
{"decomposition", unicodedata_decomposition, METH_VARARGS,
unicodedata_decomposition__doc__},
{"name", unicodedata_name, METH_VARARGS, unicodedata_name__doc__},
{"lookup", unicodedata_lookup, METH_VARARGS, unicodedata_lookup__doc__},
{"normalize", unicodedata_normalize, METH_VARARGS,
unicodedata_normalize__doc__},
{NULL, NULL} /* sentinel */
};
static PyTypeObject UCD_Type = {
/* The ob_type field must be initialized in the module init function
* to be portable to Windows without using C++. */
PyVarObject_HEAD_INIT(NULL, 0)
"unicodedata.UCD", /*tp_name*/
sizeof(PreviousDBVersion), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor)PyObject_Del, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_reserved*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
PyObject_GenericGetAttr,/*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT, /*tp_flags*/
0, /*tp_doc*/
0, /*tp_traverse*/
0, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
0, /*tp_iter*/
0, /*tp_iternext*/
unicodedata_functions, /*tp_methods*/
DB_members, /*tp_members*/
0, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
0, /*tp_init*/
0, /*tp_alloc*/
0, /*tp_new*/
0, /*tp_free*/
0, /*tp_is_gc*/
};
PyDoc_STRVAR(unicodedata_docstring,
"This module provides access to the Unicode Character Database which\n\
defines character properties for all Unicode characters. The data in\n\
this database is based on the UnicodeData.txt file version\n\
" UNIDATA_VERSION " which is publically available from ftp://ftp.unicode.org/.\n\
\n\
The module uses the same names and symbols as defined by the\n\
UnicodeData File Format " UNIDATA_VERSION ".");
static struct PyModuleDef unicodedatamodule = {
PyModuleDef_HEAD_INIT,
"unicodedata",
unicodedata_docstring,
-1,
unicodedata_functions,
NULL,
NULL,
NULL,
NULL
};
PyMODINIT_FUNC
PyInit_unicodedata(void)
{
PyObject *m, *v;
Py_TYPE(&UCD_Type) = &PyType_Type;
m = PyModule_Create(&unicodedatamodule);
if (!m)
return NULL;
PyModule_AddStringConstant(m, "unidata_version", UNIDATA_VERSION);
Py_INCREF(&UCD_Type);
PyModule_AddObject(m, "UCD", (PyObject*)&UCD_Type);
/* Previous versions */
v = new_previous_version("3.2.0", get_change_3_2_0, normalization_3_2_0);
if (v != NULL)
PyModule_AddObject(m, "ucd_3_2_0", v);
/* Export C API */
v = PyCapsule_New((void *)&hashAPI, PyUnicodeData_CAPSULE_NAME, NULL);
if (v != NULL)
PyModule_AddObject(m, "ucnhash_CAPI", v);
return m;
}
/*
Local variables:
c-basic-offset: 4
indent-tabs-mode: nil
End:
*/
Reference in New Issue View Git Blame Copy Permalink