Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

compress unicode decomposition tables (this saves another 55k)

This commit is contained in:
Fredrik Lundh 2001-01-21 22:41:08 +00:00
parent f75c9d94b4
commit 7b7dd107b3
7 changed files with 7496 additions and 10712 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Modules/ucnhash.c
View File

@ -38,7 +38,7 @@ getname(Py_UCS4 code, char* buffer, int buflen)
/* get offset into phrasebook */
offset = phrasebook_offset1[(code>>phrasebook_shift)];
offset = phrasebook_offset2[(offset<<phrasebook_shift)+
offset = phrasebook_offset2[(offset<<phrasebook_shift) +
(code&((1<<phrasebook_shift)-1))];
if (!offset)
return 0;
@ -47,13 +47,12 @@ getname(Py_UCS4 code, char* buffer, int buflen)
for (;;) {
/* get word index */
if (phrasebook[offset] & 128) {
word = phrasebook[offset] & 127;
offset++;
} else {
word = (phrasebook[offset]<<8) + phrasebook[offset+1];
offset+=2;
}
word = phrasebook[offset] - phrasebook_short;
if (word >= 0) {
word = (word << 8) + phrasebook[offset+1];
offset += 2;
} else
word = phrasebook[offset++];
if (i) {
if (i > buflen)
return 0; /* buffer overflow */

173
Modules/unicodedata.c
View File

@ -14,11 +14,40 @@
#include "Python.h"
#include "unicodedatabase.h"
typedef struct {
const unsigned char category; /* index into
_PyUnicode_CategoryNames */
const unsigned char combining; /* combining class value 0 - 255 */
const unsigned char bidirectional; /* index into
_PyUnicode_BidirectionalNames */
const unsigned char mirrored; /* true if mirrored in bidir mode */
} _PyUnicode_DatabaseRecord;
/* data file generated by Tools/unicode/makeunicodedata.py */
#include "unicodedata_db.h"
static const _PyUnicode_DatabaseRecord*
getrecord(PyUnicodeObject* v)
{
int code;
int index;
code = (int) *PyUnicode_AS_UNICODE(v);
if (code < 0 || code >= 65536)
index = 0;
else {
index = index1[(code>>SHIFT)];
index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
}
return &_PyUnicode_Database_Records[index];
}
/* --- Module API --------------------------------------------------------- */
static PyObject *
unicodedata_decimal(PyObject *self,
PyObject *args)
unicodedata_decimal(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
@ -26,18 +55,18 @@ unicodedata_decimal(PyObject *self,
if (!PyArg_ParseTuple(args, "O!|O:decimal",
&PyUnicode_Type, &v, &defobj))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
rc = Py_UNICODE_TODECIMAL(*PyUnicode_AS_UNICODE(v));
if (rc < 0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError,
"not a decimal");
goto onError;
return NULL;
}
else {
Py_INCREF(defobj);
@ -45,14 +74,10 @@ unicodedata_decimal(PyObject *self,
}
}
return PyInt_FromLong(rc);
onError:
return NULL;
}
static PyObject *
unicodedata_digit(PyObject *self,
PyObject *args)
unicodedata_digit(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
@ -60,18 +85,18 @@ unicodedata_digit(PyObject *self,
if (!PyArg_ParseTuple(args, "O!|O:digit",
&PyUnicode_Type, &v, &defobj))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
rc = Py_UNICODE_TODIGIT(*PyUnicode_AS_UNICODE(v));
if (rc < 0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError,
"not a digit");
goto onError;
return NULL;
}
else {
Py_INCREF(defobj);
@ -79,14 +104,10 @@ unicodedata_digit(PyObject *self,
}
}
return PyInt_FromLong(rc);
onError:
return NULL;
}
static PyObject *
unicodedata_numeric(PyObject *self,
PyObject *args)
unicodedata_numeric(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
PyObject *defobj = NULL;
@ -94,18 +115,18 @@ unicodedata_numeric(PyObject *self,
if (!PyArg_ParseTuple(args, "O!|O:numeric",
&PyUnicode_Type, &v, &defobj))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
rc = Py_UNICODE_TONUMERIC(*PyUnicode_AS_UNICODE(v));
if (rc < 0) {
if (defobj == NULL) {
PyErr_SetString(PyExc_ValueError,
"not a numeric character");
goto onError;
return NULL;
}
else {
Py_INCREF(defobj);
@ -113,129 +134,123 @@ unicodedata_numeric(PyObject *self,
}
}
return PyFloat_FromDouble(rc);
onError:
return NULL;
}
static PyObject *
unicodedata_category(PyObject *self,
PyObject *args)
unicodedata_category(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
if (!PyArg_ParseTuple(args, "O!:category",
&PyUnicode_Type, &v))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
index = (int) _PyUnicode_Database_GetRecord(
(int) *PyUnicode_AS_UNICODE(v)
)->category;
index = (int) getrecord(v)->category;
return PyString_FromString(_PyUnicode_CategoryNames[index]);
onError:
return NULL;
}
static PyObject *
unicodedata_bidirectional(PyObject *self,
PyObject *args)
unicodedata_bidirectional(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int index;
if (!PyArg_ParseTuple(args, "O!:bidirectional",
&PyUnicode_Type, &v))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
index = (int) _PyUnicode_Database_GetRecord(
(int) *PyUnicode_AS_UNICODE(v)
)->bidirectional;
index = (int) getrecord(v)->bidirectional;
return PyString_FromString(_PyUnicode_BidirectionalNames[index]);
onError:
return NULL;
}
static PyObject *
unicodedata_combining(PyObject *self,
PyObject *args)
unicodedata_combining(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int value;
if (!PyArg_ParseTuple(args, "O!:combining",
&PyUnicode_Type, &v))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
value = (int) _PyUnicode_Database_GetRecord(
(int) *PyUnicode_AS_UNICODE(v)
)->combining;
return PyInt_FromLong(value);
onError:
return NULL;
return PyInt_FromLong((int) getrecord(v)->combining);
}
static PyObject *
unicodedata_mirrored(PyObject *self,
PyObject *args)
unicodedata_mirrored(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
int value;
if (!PyArg_ParseTuple(args, "O!:mirrored",
&PyUnicode_Type, &v))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
value = (int) _PyUnicode_Database_GetRecord(
(int) *PyUnicode_AS_UNICODE(v)
)->mirrored;
return PyInt_FromLong(value);
onError:
return NULL;
return PyInt_FromLong((int) getrecord(v)->mirrored);
}
static PyObject *
unicodedata_decomposition(PyObject *self,
PyObject *args)
unicodedata_decomposition(PyObject *self, PyObject *args)
{
PyUnicodeObject *v;
const char *value;
char decomp[256];
int code, index, count, i;
if (!PyArg_ParseTuple(args, "O!:decomposition",
&PyUnicode_Type, &v))
goto onError;
return NULL;
if (PyUnicode_GET_SIZE(v) != 1) {
PyErr_SetString(PyExc_TypeError,
"need a single Unicode character as parameter");
goto onError;
return NULL;
}
code = (int) *PyUnicode_AS_UNICODE(v);
if (code < 0 || code >= 65536)
index = 0;
else {
index = decomp_index1[(code>>DECOMP_SHIFT)];
index = decomp_index2[(index<<DECOMP_SHIFT)+
(code&((1<<DECOMP_SHIFT)-1))];
}
/* high byte is of hex bytes (usually one or two), low byte
is prefix code (from*/
count = decomp_data[index] >> 8;
/* XXX: could allocate the PyString up front instead
(strlen(prefix) + 5 * count + 1 bytes) */
/* copy prefix */
i = strlen(decomp_prefix[decomp_data[index] & 255]);
memcpy(decomp, decomp_prefix[decomp_data[index] & 255], i);
while (count-- > 0) {
if (i)
decomp[i++] = ' ';
sprintf(decomp + i, "%04X", decomp_data[++index]);
i += strlen(decomp + i);
}
value = _PyUnicode_Database_GetDecomposition(
(int) *PyUnicode_AS_UNICODE(v)
);
return PyString_FromString(value);
onError:
return NULL;
decomp[i] = '\0';
return PyString_FromString(decomp);
}
/* XXX Add doc strings. */

4916
Modules/unicodedata_db.h
View File

File diff suppressed because it is too large Load Diff

48
Modules/unicodedatabase.c
View File

@ -1,47 +1 @@
/* ------------------------------------------------------------------------
unicodedatabase -- The Unicode 3.0 data base.
Data was extracted from the Unicode 3.0 UnicodeData.txt file.
Written by Marc-Andre Lemburg (mal@lemburg.com).
Rewritten for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
Copyright (c) Corporation for National Research Initiatives.
------------------------------------------------------------------------ */
#include "Python.h"
#include "unicodedatabase.h"
/* read the actual data from a separate file! */
#include "unicodedata_db.h"
const _PyUnicode_DatabaseRecord *
_PyUnicode_Database_GetRecord(int code)
{
int index;
if (code < 0 || code >= 65536)
index = 0;
else {
index = index1[(code>>SHIFT)];
index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
}
return &_PyUnicode_Database_Records[index];
}
const char *
_PyUnicode_Database_GetDecomposition(int code)
{
int index;
if (code < 0 || code >= 65536)
index = 0;
else {
index = decomp_index1[(code>>DECOMP_SHIFT)];
index = decomp_index2[(index<<DECOMP_SHIFT)+
(code&((1<<DECOMP_SHIFT)-1))];
}
return decomp_data[index];
}
/* remove this file! */

34
Modules/unicodedatabase.h
View File

@ -1,33 +1 @@
/* ------------------------------------------------------------------------
unicodedatabase -- The Unicode 3.0 data base.
Data was extracted from the Unicode 3.0 UnicodeData.txt file.
Written by Marc-Andre Lemburg (mal@lemburg.com).
Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
Copyright (c) Corporation for National Research Initiatives.
------------------------------------------------------------------------ */
/* --- Unicode database entry --------------------------------------------- */
typedef struct {
const unsigned char category; /* index into
_PyUnicode_CategoryNames */
const unsigned char combining; /* combining class value 0 - 255 */
const unsigned char bidirectional; /* index into
_PyUnicode_BidirectionalNames */
const unsigned char mirrored; /* true if mirrored in bidir mode */
} _PyUnicode_DatabaseRecord;
/* --- Unicode category names --------------------------------------------- */
extern const char *_PyUnicode_CategoryNames[];
extern const char *_PyUnicode_BidirectionalNames[];
/* --- Unicode Database --------------------------------------------------- */
extern const _PyUnicode_DatabaseRecord *_PyUnicode_Database_GetRecord(int ch);
extern const char *_PyUnicode_Database_GetDecomposition(int ch);
/* remove this file! */

12905
Modules/unicodename_db.h
View File

File diff suppressed because it is too large Load Diff

117
Tools/unicode/makeunicodedata.py
View File

@ -12,8 +12,9 @@
# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl expand first/last ranges
# 2001-01-19 fl added character name tables (2.1)
# 2001-01-21 fl added decomp compression; dynamic phrasebook threshold
#
# written by Fredrik Lundh (fredrik@pythonware.com), September 2000
# written by Fredrik Lundh (fredrik@pythonware.com)
#
import sys
@ -50,9 +51,9 @@ def maketables(trace=0):
print len(filter(None, unicode.table)), "characters"
# makeunicodename(unicode, trace)
makeunicodedata(unicode, trace)
makeunicodetype(unicode, trace)
makeunicodename(unicode, trace)
# makeunicodetype(unicode, trace)
# --------------------------------------------------------------------
# unicode character properties
@ -90,27 +91,45 @@ def makeunicodedata(unicode, trace):
# 2) decomposition data
# FIXME: <fl> using the encoding stuff from unidb would save
# another 50k or so, but I'll leave that for 2.1...
decomp_data = [""]
decomp_data = [0]
decomp_prefix = [""]
decomp_index = [0] * len(unicode.chars)
decomp_size = 0
for char in unicode.chars:
record = unicode.table[char]
if record:
if record[5]:
decomp = string.split(record[5])
# prefix
if decomp[0][0] == "<":
prefix = decomp.pop(0)
else:
prefix = ""
try:
i = decomp_data.index(record[5])
i = decomp_prefix.index(prefix)
except ValueError:
i = len(decomp_prefix)
decomp_prefix.append(prefix)
prefix = i
assert prefix < 256
# content
decomp = [prefix + (len(decomp)<<8)] +\
map(lambda s: int(s, 16), decomp)
try:
i = decomp_data.index(decomp)
except ValueError:
i = len(decomp_data)
decomp_data.append(record[5])
decomp_data.extend(decomp)
decomp_size = decomp_size + len(decomp) * 2
else:
i = 0
decomp_index[char] = i
print len(table), "unique properties"
print len(decomp_data), "unique decomposition entries"
print len(decomp_prefix), "unique decomposition prefixes"
print len(decomp_data), "unique decomposition entries:",
print decomp_size, "bytes"
print "--- Writing", FILE, "..."
@ -141,8 +160,8 @@ def makeunicodedata(unicode, trace):
print >>fp, " NULL"
print >>fp, "};"
print >>fp, "static const char *decomp_data[] = {"
for name in decomp_data:
print >>fp, "static const char *decomp_prefix[] = {"
for name in decomp_prefix:
print >>fp, " \"%s\"," % name
print >>fp, " NULL"
print >>fp, "};"
@ -152,16 +171,19 @@ def makeunicodedata(unicode, trace):
print >>fp, "/* index tables for the database records */"
print >>fp, "#define SHIFT", shift
Array("index1", index1).dump(fp)
Array("index2", index2).dump(fp)
Array("index1", index1).dump(fp, trace)
Array("index2", index2).dump(fp, trace)
# split decomposition index table
index1, index2, shift = splitbins(decomp_index, trace)
print >>fp, "/* decomposition data */"
Array("decomp_data", decomp_data).dump(fp, trace)
print >>fp, "/* index tables for the decomposition data */"
print >>fp, "#define DECOMP_SHIFT", shift
Array("decomp_index1", index1).dump(fp)
Array("decomp_index2", index2).dump(fp)
Array("decomp_index1", index1).dump(fp, trace)
Array("decomp_index2", index2).dump(fp, trace)
fp.close()
@ -250,8 +272,8 @@ def makeunicodetype(unicode, trace):
print >>fp, "/* type indexes */"
print >>fp, "#define SHIFT", shift
Array("index1", index1).dump(fp)
Array("index2", index2).dump(fp)
Array("index1", index1).dump(fp, trace)
Array("index2", index2).dump(fp, trace)
fp.close()
@ -302,16 +324,28 @@ def makeunicodename(unicode, trace):
# sort on falling frequency
wordlist.sort(lambda a, b: len(b[1])-len(a[1]))
# figure out how many phrasebook escapes we need
escapes = 0
while escapes * 256 < len(wordlist):
escapes = escapes + 1
print escapes, "escapes"
short = 256 - escapes
assert short > 0
print short, "short indexes in lexicon"
# statistics
n = 0
for i in range(128):
for i in range(short):
n = n + len(wordlist[i][1])
print n, "short words (7-bit indices)"
print n, "short indexes in phrasebook"
# pick the 128 most commonly used words, and sort the rest on
# falling length (to maximize overlap)
# pick the most commonly used words, and sort the rest on falling
# length (to maximize overlap)
wordlist, wordtail = wordlist[:128], wordlist[128:]
wordlist, wordtail = wordlist[:short], wordlist[short:]
wordtail.sort(lambda a, b: len(b[0])-len(a[0]))
wordlist.extend(wordtail)
@ -334,11 +368,7 @@ def makeunicodename(unicode, trace):
lexicon = lexicon + ww
offset = offset + len(w)
words[w] = len(lexicon_offset)
lexicon_offset.append(offset)
print len(words), "words in lexicon;", len(lexicon), "bytes"
assert len(words) < 32768 # 15-bit word indices
lexicon_offset.append(o)
lexicon = map(ord, lexicon)
@ -352,12 +382,15 @@ def makeunicodename(unicode, trace):
phrasebook_offset[char] = len(phrasebook)
for w in w:
i = words[w]
if i < 128:
phrasebook.append(128+i)
if i < short:
phrasebook.append(i)
else:
phrasebook.append(i>>8)
# store as two bytes
phrasebook.append((i>>8) + short)
phrasebook.append(i&255)
assert getsize(phrasebook) == 1
#
# unicode name hash table
@ -384,21 +417,22 @@ def makeunicodename(unicode, trace):
print >>fp, "#define NAME_MAXLEN", 256
print >>fp
print >>fp, "/* lexicon */"
Array("lexicon", lexicon).dump(fp)
Array("lexicon_offset", lexicon_offset).dump(fp)
Array("lexicon", lexicon).dump(fp, trace)
Array("lexicon_offset", lexicon_offset).dump(fp, trace)
# split decomposition index table
offset1, offset2, shift = splitbins(phrasebook_offset, trace)
print >>fp, "/* code->name phrasebook */"
print >>fp, "#define phrasebook_shift", shift
print >>fp, "#define phrasebook_short", short
Array("phrasebook", phrasebook).dump(fp)
Array("phrasebook_offset1", offset1).dump(fp)
Array("phrasebook_offset2", offset2).dump(fp)
Array("phrasebook", phrasebook).dump(fp, trace)
Array("phrasebook_offset1", offset1).dump(fp, trace)
Array("phrasebook_offset2", offset2).dump(fp, trace)
print >>fp, "/* name->code dictionary */"
codehash.dump(fp)
codehash.dump(fp, trace)
fp.close()
@ -527,9 +561,9 @@ class Hash:
self.size = size
self.poly = poly
def dump(self, file):
def dump(self, file, trace):
# write data to file, as a C array
self.data.dump(file)
self.data.dump(file, trace)
file.write("#define %s_magic %d\n" % (self.name, self.magic))
file.write("#define %s_size %d\n" % (self.name, self.size))
file.write("#define %s_poly %d\n" % (self.name, self.poly))
@ -542,10 +576,11 @@ class Array:
self.name = name
self.data = data
def dump(self, file):
def dump(self, file, trace=0):
# write data to file, as a C array
size = getsize(self.data)
# print >>sys.stderr, self.name+":", size*len(self.data), "bytes"
if trace:
print >>sys.stderr, self.name+":", size*len(self.data), "bytes"
file.write("static ")
if size == 1:
file.write("unsigned char")