Port normalization to new API.
This commit is contained in:
Martin v. Löwis
parent
f0ddadcf2e
commit
22970667da
|
|
@ -494,36 +494,44 @@ static PyObject*
|
|||
nfd_nfkd(PyObject *self, PyObject *input, int k)
|
||||
{
|
||||
PyObject *result;
|
||||
Py_UNICODE *i, *end, *o;
|
||||
Py_UCS4 *output;
|
||||
Py_ssize_t i, o, osize;
|
||||
int kind;
|
||||
void *data;
|
||||
/* Longest decomposition in Unicode 3.2: U+FDFA */
|
||||
Py_UNICODE stack[20];
|
||||
Py_UCS4 stack[20];
|
||||
Py_ssize_t space, isize;
|
||||
int index, prefix, count, stackptr;
|
||||
unsigned char prev, cur;
|
||||
|
||||
stackptr = 0;
|
||||
isize = PyUnicode_GET_SIZE(input);
|
||||
isize = PyUnicode_GET_LENGTH(input);
|
||||
/* Overallocate atmost 10 characters. */
|
||||
space = (isize > 10 ? 10 : isize) + isize;
|
||||
result = PyUnicode_FromUnicode(NULL, space);
|
||||
if (!result)
|
||||
osize = space;
|
||||
output = PyMem_Malloc(space * sizeof(Py_UCS4));
|
||||
if (!output) {
|
||||
PyErr_NoMemory();
|
||||
return NULL;
|
||||
i = PyUnicode_AS_UNICODE(input);
|
||||
end = i + isize;
|
||||
o = PyUnicode_AS_UNICODE(result);
|
||||
}
|
||||
i = o = 0;
|
||||
kind = PyUnicode_KIND(input);
|
||||
data = PyUnicode_DATA(input);
|
||||
|
||||
while (i < end) {
|
||||
stack[stackptr++] = *i++;
|
||||
while (i < isize) {
|
||||
stack[stackptr++] = PyUnicode_READ(kind, data, i++);
|
||||
while(stackptr) {
|
||||
Py_UNICODE code = stack[--stackptr];
|
||||
Py_UCS4 code = stack[--stackptr];
|
||||
/* Hangul Decomposition adds three characters in
|
||||
a single step, so we need atleast that much room. */
|
||||
if (space < 3) {
|
||||
Py_ssize_t newsize = PyUnicode_GET_SIZE(result) + 10;
|
||||
osize += 10;
|
||||
space += 10;
|
||||
if (PyUnicode_Resize(&result, newsize) == -1)
|
||||
output = PyMem_Realloc(output, osize*sizeof(Py_UCS4));
|
||||
if (output == NULL) {
|
||||
PyErr_NoMemory();
|
||||
return NULL;
|
||||
o = PyUnicode_AS_UNICODE(result) + newsize - space;
|
||||
}
|
||||
}
|
||||
/* Hangul Decomposition. */
|
||||
if (SBase <= code && code < (SBase+SCount)) {
|
||||
|
|
@ -531,11 +539,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
|
|||
int L = LBase + SIndex / NCount;
|
||||
int V = VBase + (SIndex % NCount) / TCount;
|
||||
int T = TBase + SIndex % TCount;
|
||||
*o++ = L;
|
||||
*o++ = V;
|
||||
output[o++] = L;
|
||||
output[o++] = V;
|
||||
space -= 2;
|
||||
if (T != TBase) {
|
||||
*o++ = T;
|
||||
output[o++] = T;
|
||||
space --;
|
||||
}
|
||||
continue;
|
||||
|
|
@ -555,7 +563,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
|
|||
/* Copy character if it is not decomposable, or has a
|
||||
compatibility decomposition, but we do NFD. */
|
||||
if (!count || (prefix && !k)) {
|
||||
*o++ = code;
|
||||
output[o++] = code;
|
||||
space--;
|
||||
continue;
|
||||
}
|
||||
|
|
@ -568,15 +576,20 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
|
|||
}
|
||||
}
|
||||
|
||||
/* Drop overallocation. Cannot fail. */
|
||||
PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space);
|
||||
result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
|
||||
output, o);
|
||||
PyMem_Free(output);
|
||||
if (!result)
|
||||
return NULL;
|
||||
/* result is guaranteed to be ready, as it is compact. */
|
||||
kind = PyUnicode_KIND(result);
|
||||
data = PyUnicode_DATA(result);
|
||||
|
||||
/* Sort canonically. */
|
||||
i = PyUnicode_AS_UNICODE(result);
|
||||
prev = _getrecord_ex(*i)->combining;
|
||||
end = i + PyUnicode_GET_SIZE(result);
|
||||
for (i++; i < end; i++) {
|
||||
cur = _getrecord_ex(*i)->combining;
|
||||
i = 0;
|
||||
prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
|
||||
for (i++; i < PyUnicode_GET_LENGTH(result); i++) {
|
||||
cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
|
||||
if (prev == 0 || cur == 0 || prev <= cur) {
|
||||
prev = cur;
|
||||
continue;
|
||||
|
|
@ -584,23 +597,24 @@ nfd_nfkd(PyObject *self, PyObject *input, int k)
|
|||
/* Non-canonical order. Need to switch *i with previous. */
|
||||
o = i - 1;
|
||||
while (1) {
|
||||
Py_UNICODE tmp = o[1];
|
||||
o[1] = o[0];
|
||||
o[0] = tmp;
|
||||
Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1);
|
||||
PyUnicode_WRITE(kind, data, o+1,
|
||||
PyUnicode_READ(kind, data, o));
|
||||
PyUnicode_WRITE(kind, data, o, tmp);
|
||||
o--;
|
||||
if (o < PyUnicode_AS_UNICODE(result))
|
||||
if (o < 0)
|
||||
break;
|
||||
prev = _getrecord_ex(*o)->combining;
|
||||
prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining;
|
||||
if (prev == 0 || prev <= cur)
|
||||
break;
|
||||
}
|
||||
prev = _getrecord_ex(*i)->combining;
|
||||
prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static int
|
||||
find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code)
|
||||
find_nfc_index(PyObject *self, struct reindex* nfc, Py_UCS4 code)
|
||||
{
|
||||
int index;
|
||||
for (index = 0; nfc[index].start; index++) {
|
||||
|
|
@ -619,27 +633,36 @@ static PyObject*
|
|||
nfc_nfkc(PyObject *self, PyObject *input, int k)
|
||||
{
|
||||
PyObject *result;
|
||||
Py_UNICODE *i, *i1, *o, *end;
|
||||
int kind;
|
||||
void *data;
|
||||
Py_UCS4 *output;
|
||||
Py_ssize_t i, i1, o, len;
|
||||
int f,l,index,index1,comb;
|
||||
Py_UNICODE code;
|
||||
Py_UNICODE *skipped[20];
|
||||
Py_UCS4 code;
|
||||
Py_ssize_t skipped[20];
|
||||
int cskipped = 0;
|
||||
|
||||
result = nfd_nfkd(self, input, k);
|
||||
if (!result)
|
||||
return NULL;
|
||||
/* result will be "ready". */
|
||||
kind = PyUnicode_KIND(result);
|
||||
data = PyUnicode_DATA(result);
|
||||
len = PyUnicode_GET_LENGTH(result);
|
||||
|
||||
/* We are going to modify result in-place.
|
||||
If nfd_nfkd is changed to sometimes return the input,
|
||||
this code needs to be reviewed. */
|
||||
assert(result != input);
|
||||
|
||||
i = PyUnicode_AS_UNICODE(result);
|
||||
end = i + PyUnicode_GET_SIZE(result);
|
||||
o = PyUnicode_AS_UNICODE(result);
|
||||
/* We allocate a buffer for the output.
|
||||
If we find that we made no changes, we still return
|
||||
the NFD result. */
|
||||
output = PyMem_Malloc(len * sizeof(Py_UCS4));
|
||||
if (!output) {
|
||||
PyErr_NoMemory();
|
||||
Py_DECREF(result);
|
||||
return 0;
|
||||
}
|
||||
i = o = 0;
|
||||
|
||||
again:
|
||||
while (i < end) {
|
||||
while (i < len) {
|
||||
for (index = 0; index < cskipped; index++) {
|
||||
if (skipped[index] == i) {
|
||||
/* *i character is skipped.
|
||||
|
|
@ -652,33 +675,41 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
|
|||
}
|
||||
/* Hangul Composition. We don't need to check for <LV,T>
|
||||
pairs, since we always have decomposed data. */
|
||||
if (LBase <= *i && *i < (LBase+LCount) &&
|
||||
i + 1 < end &&
|
||||
VBase <= i[1] && i[1] <= (VBase+VCount)) {
|
||||
code = PyUnicode_READ(kind, data, i);
|
||||
if (LBase <= code && code < (LBase+LCount) &&
|
||||
i + 1 < len &&
|
||||
VBase <= PyUnicode_READ(kind, data, i+1) &&
|
||||
PyUnicode_READ(kind, data, i+1) <= (VBase+VCount)) {
|
||||
int LIndex, VIndex;
|
||||
LIndex = i[0] - LBase;
|
||||
VIndex = i[1] - VBase;
|
||||
LIndex = code - LBase;
|
||||
VIndex = PyUnicode_READ(kind, data, i+1) - VBase;
|
||||
code = SBase + (LIndex*VCount+VIndex)*TCount;
|
||||
i+=2;
|
||||
if (i < end &&
|
||||
TBase <= *i && *i <= (TBase+TCount)) {
|
||||
code += *i-TBase;
|
||||
if (i < len &&
|
||||
TBase <= PyUnicode_READ(kind, data, i) &&
|
||||
PyUnicode_READ(kind, data, i) <= (TBase+TCount)) {
|
||||
code += PyUnicode_READ(kind, data, i)-TBase;
|
||||
i++;
|
||||
}
|
||||
*o++ = code;
|
||||
output[o++] = code;
|
||||
continue;
|
||||
}
|
||||
|
||||
f = find_nfc_index(self, nfc_first, *i);
|
||||
/* code is still input[i] here */
|
||||
f = find_nfc_index(self, nfc_first, code);
|
||||
if (f == -1) {
|
||||
*o++ = *i++;
|
||||
output[o++] = code;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
/* Find next unblocked character. */
|
||||
i1 = i+1;
|
||||
comb = 0;
|
||||
while (i1 < end) {
|
||||
int comb1 = _getrecord_ex(*i1)->combining;
|
||||
/* output base character for now; might be updated later. */
|
||||
output[o] = PyUnicode_READ(kind, data, i);
|
||||
while (i1 < len) {
|
||||
Py_UCS4 code1 = PyUnicode_READ(kind, data, i1);
|
||||
int comb1 = _getrecord_ex(code1)->combining;
|
||||
if (comb) {
|
||||
if (comb1 == 0)
|
||||
break;
|
||||
|
|
@ -688,8 +719,8 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
|
|||
continue;
|
||||
}
|
||||
}
|
||||
l = find_nfc_index(self, nfc_last, *i1);
|
||||
/* *i1 cannot be combined with *i. If *i1
|
||||
l = find_nfc_index(self, nfc_last, code1);
|
||||
/* i1 cannot be combined with i. If i1
|
||||
is a starter, we don't need to look further.
|
||||
Otherwise, record the combining class. */
|
||||
if (l == -1) {
|
||||
|
|
@ -708,19 +739,28 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
|
|||
goto not_combinable;
|
||||
|
||||
/* Replace the original character. */
|
||||
*i = code;
|
||||
output[o] = code;
|
||||
/* Mark the second character unused. */
|
||||
assert(cskipped < 20);
|
||||
skipped[cskipped++] = i1;
|
||||
i1++;
|
||||
f = find_nfc_index(self, nfc_first, *i);
|
||||
f = find_nfc_index(self, nfc_first, output[o]);
|
||||
if (f == -1)
|
||||
break;
|
||||
}
|
||||
*o++ = *i++;
|
||||
/* Output character was already written.
|
||||
Just advance the indices. */
|
||||
o++; i++;
|
||||
}
|
||||
if (o != end)
|
||||
PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result));
|
||||
if (o == len) {
|
||||
/* No changes. Return original string. */
|
||||
PyMem_Free(output);
|
||||
return result;
|
||||
}
|
||||
Py_DECREF(result);
|
||||
result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
|
||||
output, o);
|
||||
PyMem_Free(output);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -728,7 +768,9 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
|
|||
static int
|
||||
is_normalized(PyObject *self, PyObject *input, int nfc, int k)
|
||||
{
|
||||
Py_UNICODE *i, *end;
|
||||
Py_ssize_t i, len;
|
||||
int kind;
|
||||
void *data;
|
||||
unsigned char prev_combining = 0, quickcheck_mask;
|
||||
|
||||
/* An older version of the database is requested, quickchecks must be
|
||||
|
|
@ -740,10 +782,13 @@ is_normalized(PyObject *self, PyObject *input, int nfc, int k)
|
|||
as described in http://unicode.org/reports/tr15/#Annex8. */
|
||||
quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
|
||||
|
||||
i = PyUnicode_AS_UNICODE(input);
|
||||
end = i + PyUnicode_GET_SIZE(input);
|
||||
while (i < end) {
|
||||
const _PyUnicode_DatabaseRecord *record = _getrecord_ex(*i++);
|
||||
i = 0;
|
||||
kind = PyUnicode_KIND(input);
|
||||
data = PyUnicode_DATA(input);
|
||||
len = PyUnicode_GET_LENGTH(input);
|
||||
while (i < len) {
|
||||
Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
|
||||
const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
|
||||
unsigned char combining = record->combining;
|
||||
unsigned char quickcheck = record->normalization_quick_check;
|
||||
|
||||
|
|
@ -772,7 +817,10 @@ unicodedata_normalize(PyObject *self, PyObject *args)
|
|||
&form, &PyUnicode_Type, &input))
|
||||
return NULL;
|
||||
|
||||
if (PyUnicode_GetSize(input) == 0) {
|
||||
if (PyUnicode_READY(input) == -1)
|
||||
return NULL;
|
||||
|
||||
if (PyUnicode_GET_LENGTH(input) == 0) {
|
||||
/* Special case empty input strings, since resizing
|
||||
them later would cause internal errors. */
|
||||
Py_INCREF(input);
|
||||
|
|
|
|||
Loading…
Reference in New Issue