traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
cpython/Tools/perfecthash/perfect_hash.py

665 lines
21 KiB
Python
Raw Blame History

#!/usr/bin/env/python
# perfect_hash.py
#
# Outputs C code for a minimal perfect hash.
# The hash is produced using the algorithm described in
# "Optimal algorithms for minimal perfect hashing",
# G. Havas, B.S. Majewski. Available as a technical report
# from the CS department, University of Queensland
# (ftp://ftp.cs.uq.oz.au/).
#
# This is a modified version of Andrew Kuchling's code
# (http://starship.python.net/crew/amk/python/code/perfect-hash.html)
# and generates C fragments suitable for compilation as a Python
# extension module.
#
# Difference between this algorithm and gperf:
# Gperf will complete in finite time with a successful function,
# or by giving up.
# This algorithm may never complete, although it is extremely likely
# when c >= 2.
# The algorithm works like this:
# 0) You have K keys, that you want to perfectly hash to a bunch
# of hash values.
#
# 1) Choose a number N larger than K. This is the number of
# vertices in a graph G, and also the size of the resulting table.
#
# 2) Pick two random hash functions f1, f2, that output values from
# 0...N-1.
#
# 3) for key in keys:
# h1 = f1(key) ; h2 = f2(key)
# Draw an edge between vertices h1 and h2 of the graph.
# Associate the desired hash value with that edge.
#
# 4) Check if G is acyclic; if not, go back to step 1 and pick a bigger N.
#
# 5) Assign values to each vertex such that, for each edge, you can
# add the values for the two vertices and get the desired value
# for that edge -- which is the desired hash key. This task is
# dead easy, because the graph is acyclic. This is done by
# picking a vertex V, and assigning it a value of 0. You then do a
# depth-first search, assigning values to new vertices so that
# they sum up properly.
#
# 6) f1, f2, and G now make up your perfect hash function.
import sys, whrandom, string
import pprint
import perfhash
import time
class Hash:
"""Random hash function
For simplicity and speed, this doesn't implement any byte-level hashing
scheme. Instead, a random string is generated and prefixing to
str(key), and then Python's hashing function is used."""
def __init__(self, N, caseInsensitive=0):
self.N = N
junk = ""
for i in range(10):
junk = junk + whrandom.choice(string.letters + string.digits)
self.junk = junk
self.caseInsensitive = caseInsensitive
self.seed = perfhash.calcSeed(junk)
def __call__(self, key):
key = str(key)
if self.caseInsensitive:
key = string.upper(key)
x = perfhash.hash(self.seed, len(self.junk), key) % self.N
#h = hash(self.junk + key) % self.N
#assert x == h
return x
def generate_code(self):
s = """{
register int len;
register unsigned char *p;
register long x;
len = cch;
p = (unsigned char *) key;
x = %(junkSeed)d;
while (--len >= 0)
x = (1000003*x) ^ """ % \
{
"lenJunk" : len(self.junk),
"junkSeed" : self.seed,
}
if self.caseInsensitive:
s = s + "toupper(*(p++));"
else:
s = s + "*(p++);"
s = s + """
x ^= cch + %(lenJunk)d;
if (x == -1)
x = -2;
x %%= k_cHashElements;
/* ensure the returned value is positive so we mimic Python's %% operator */
if (x < 0)
x += k_cHashElements;
return x;
}
""" % { "lenJunk" : len(self.junk),
"junkSeed" : self.seed, }
return s
WHITE, GREY, BLACK = 0,1,2
class Graph:
"""Graph class. This class isn't particularly efficient or general,
and only has the features I needed to implement this algorithm.
num_vertices -- number of vertices
edges -- maps 2-tuples of vertex numbers to the value for this
edge. If there's an edge between v1 and v2 (v1<v2),
(v1,v2) is a key and the value is the edge's value.
reachable_list -- maps a vertex V to the list of vertices
to which V is connected by edges. Used
for traversing the graph.
values -- numeric value for each vertex
"""
def __init__(self, num_vertices):
self.num_vertices = num_vertices
self.edges = {}
self.reachable_list = {}
self.values = [-1] * num_vertices
def connect(self, vertex1, vertex2, value):
"""Connect 'vertex1' and 'vertex2' with an edge, with associated
value 'value'"""
if vertex1 > vertex2: vertex1, vertex2 = vertex2, vertex1
# if self.edges.has_key( (vertex1, vertex2) ):
# raise ValueError, 'Collision: vertices already connected'
self.edges[ (vertex1, vertex2) ] = value
# Add vertices to each other's reachable list
if not self.reachable_list.has_key( vertex1 ):
self.reachable_list[ vertex1 ] = [vertex2]
else:
self.reachable_list[vertex1].append(vertex2)
if not self.reachable_list.has_key( vertex2 ):
self.reachable_list[ vertex2 ] = [vertex1]
else:
self.reachable_list[vertex2].append(vertex1)
def get_edge_value(self, vertex1, vertex2):
"""Retrieve the value corresponding to the edge between
'vertex1' and 'vertex2'. Raises KeyError if no such edge"""
if vertex1 > vertex2:
vertex1, vertex2 = vertex2, vertex1
return self.edges[ (vertex1, vertex2) ]
def is_acyclic(self):
"Returns true if the graph is acyclic, otherwise false"
# This is done by doing a depth-first search of the graph;
# painting each vertex grey and then black. If the DFS
# ever finds a vertex that isn't white, there's a cycle.
colour = {}
for i in range(self.num_vertices): colour[i] = WHITE
# Loop over all vertices, taking white ones as starting
# points for a traversal.
for i in range(self.num_vertices):
if colour[i] == WHITE:
# List of vertices to visit
visit_list = [ (None,i) ]
# Do a DFS
while visit_list:
# Colour this vertex grey.
parent, vertex = visit_list[0] ; del visit_list[0]
colour[vertex] = GREY
# Make copy of list of neighbours, removing the vertex
# we arrived here from.
neighbours = self.reachable_list.get(vertex, []) [:]
if parent in neighbours: neighbours.remove( parent )
for neighbour in neighbours:
if colour[neighbour] == WHITE:
visit_list.insert(0, (vertex, neighbour) )
elif colour[neighbour] != WHITE:
# Aha! Already visited this node,
# so the graph isn't acyclic.
return 0
colour[vertex] = BLACK
# We got through, so the graph is acyclic.
return 1
def assign_values(self):
"""Compute values for each vertex, so that they sum up
properly to the associated value for each edge."""
# Also done with a DFS; I simply copied the DFS code
# from is_acyclic(). (Should generalize the logic so
# one function could be used from both methods,
# but I couldn't be bothered.)
colour = {}
for i in range(self.num_vertices): colour[i] = WHITE
# Loop over all vertices, taking white ones as starting
# points for a traversal.
for i in range(self.num_vertices):
if colour[i] == WHITE:
# Set this vertex's value, arbitrarily, to zero.
self.set_vertex_value( i, 0 )
# List of vertices to visit
visit_list = [ (None,i) ]
# Do a DFS
while visit_list:
# Colour this vertex grey.
parent, vertex = visit_list[0] ; del visit_list[0]
colour[vertex] = GREY
# Make copy of list of neighbours, removing the vertex
# we arrived here from.
neighbours = self.reachable_list.get(vertex, []) [:]
if parent in neighbours: neighbours.remove( parent )
for neighbour in self.reachable_list.get(vertex, []):
edge_value = self.get_edge_value( vertex, neighbour )
if colour[neighbour] == WHITE:
visit_list.insert(0, (vertex, neighbour) )
# Set new vertex's value to the desired
# edge value, minus the value of the
# vertex we came here from.
new_val = (edge_value -
self.get_vertex_value( vertex ) )
self.set_vertex_value( neighbour,
new_val % self.num_vertices)
colour[vertex] = BLACK
# Returns nothing
return
def __getitem__(self, index):
if index < self.num_vertices: return index
raise IndexError
def get_vertex_value(self, vertex):
"Get value for a vertex"
return self.values[ vertex ]
def set_vertex_value(self, vertex, value):
"Set value for a vertex"
self.values[ vertex ] = value
def generate_code(self, out, width = 70):
"Return nicely formatted table"
out.write("{ ")
pos = 0
for v in self.values:
v=str(v)+', '
out.write(v)
pos = pos + len(v) + 1
if pos > width: out.write('\n '); pos = 0
out.write('};\n')
class PerfectHash:
def __init__(self, cchMax, f1, f2, G, cHashElements, cKeys, maxHashValue):
self.cchMax = cchMax
self.f1 = f1
self.f2 = f2
self.G = G
self.cHashElements = cHashElements
self.cKeys = cKeys
# determine the necessary type for storing our hash function
# helper table:
self.type = self.determineType(maxHashValue)
def generate_header(self, structName):
header = """
#include <Python.h>
#include <stdlib.h>
/* --- C API ----------------------------------------------------*/
/* C API for usage by other Python modules */
typedef struct %(structName)s
{
unsigned long cKeys;
unsigned long cchMax;
unsigned long (*hash)(const char *key, unsigned int cch);
const void *(*getValue)(unsigned long iKey);
} %(structName)s;
""" % { "structName" : structName }
return header
def determineType(self, maxHashValue):
if maxHashValue <= 255:
return "unsigned char"
elif maxHashValue <= 65535:
return "unsigned short"
else:
# Take the cheesy way out...
return "unsigned long"
def generate_code(self, moduleName, dataArrayName, dataArrayType, structName):
# Output C code for the hash functions and tables
code = """
/*
* The hash is produced using the algorithm described in
* "Optimal algorithms for minimal perfect hashing",
* G. Havas, B.S. Majewski. Available as a technical report
* from the CS department, University of Queensland
* (ftp://ftp.cs.uq.oz.au/).
*
* Generated using a heavily tweaked version of Andrew Kuchling's
* perfect_hash.py:
* http://starship.python.net/crew/amk/python/code/perfect-hash.html
*
* Generated on: %s
*/
""" % time.ctime(time.time())
# MSVC SP3 was complaining when I actually used a global constant
code = code + """
#define k_cHashElements %i
#define k_cchMaxKey %d
#define k_cKeys %i
""" % (self.cHashElements, self.cchMax, self.cKeys)
code = code + """
static const %s G[k_cHashElements];
static const %s %s[k_cKeys];
""" % (self.type, dataArrayType, dataArrayName)
code = code + """
static long f1(const char *key, unsigned int cch)
"""
code = code + self.f1.generate_code()
code = code + """
static long f2(const char *key, unsigned int cch)
"""
code = code + self.f2.generate_code()
code = code + """
static unsigned long hash(const char *key, unsigned int cch)
{
return ((unsigned long)(G[ f1(key, cch) ]) + (unsigned long)(G[ f2(key, cch) ]) ) %% k_cHashElements;
}
const void *getValue(unsigned long iKey)
{
return &%(dataArrayName)s[iKey];
}
/* Helper for adding objects to dictionaries. Check for errors with
PyErr_Occurred() */
static
void insobj(PyObject *dict,
char *name,
PyObject *v)
{
PyDict_SetItemString(dict, name, v);
Py_XDECREF(v);
}
static const %(structName)s hashAPI =
{
k_cKeys,
k_cchMaxKey,
&hash,
&getValue,
};
static
PyMethodDef Module_methods[] =
{
{NULL, NULL},
};
static char *Module_docstring = "%(moduleName)s hash function module";
/* Error reporting for module init functions */
#define Py_ReportModuleInitError(modname) { \\
PyObject *exc_type, *exc_value, *exc_tb; \\
PyObject *str_type, *str_value; \\
\\
/* Fetch error objects and convert them to strings */ \\
PyErr_Fetch(&exc_type, &exc_value, &exc_tb); \\
if (exc_type && exc_value) { \\
str_type = PyObject_Str(exc_type); \\
str_value = PyObject_Str(exc_value); \\
} \\
else { \\
str_type = NULL; \\
str_value = NULL; \\
} \\
/* Try to format a more informative error message using the \\
original error */ \\
if (str_type && str_value && \\
PyString_Check(str_type) && PyString_Check(str_value)) \\
PyErr_Format( \\
PyExc_ImportError, \\
"initialization of module "modname" failed " \\
"(%%s:%%s)", \\
PyString_AS_STRING(str_type), \\
PyString_AS_STRING(str_value)); \\
else \\
PyErr_SetString( \\
PyExc_ImportError, \\
"initialization of module "modname" failed"); \\
Py_XDECREF(str_type); \\
Py_XDECREF(str_value); \\
Py_XDECREF(exc_type); \\
Py_XDECREF(exc_value); \\
Py_XDECREF(exc_tb); \\
}
/* Create PyMethodObjects and register them in the module\'s dict */
DL_EXPORT(void)
init%(moduleName)s(void)
{
PyObject *module, *moddict;
/* Create module */
module = Py_InitModule4("%(moduleName)s", /* Module name */
Module_methods, /* Method list */
Module_docstring, /* Module doc-string */
(PyObject *)NULL, /* always pass this as *self */
PYTHON_API_VERSION); /* API Version */
if (module == NULL)
goto onError;
/* Add some constants to the module\'s dict */
moddict = PyModule_GetDict(module);
if (moddict == NULL)
goto onError;
/* Export C API */
insobj(
moddict,
"%(moduleName)sAPI",
PyCObject_FromVoidPtr((void *)&hashAPI, NULL));
onError:
/* Check for errors and report them */
if (PyErr_Occurred())
Py_ReportModuleInitError("%(moduleName)s");
return;
}
""" % { "moduleName" : moduleName,
"dataArrayName" : dataArrayName,
"structName" : structName, }
return code
def generate_graph(self, out):
out.write("""
static const unsigned short G[] =
""")
self.G.generate_code(out)
def generate_hash(keys, caseInsensitive=0,
minC=None, initC=None,
f1Seed=None, f2Seed=None,
cIncrement=None, cTries=None):
"""Print out code for a perfect minimal hash. Input is a list of
(key, desired hash value) tuples. """
# K is the number of keys.
K = len(keys)
# We will be generating graphs of size N, where N = c * K.
# The larger C is, the fewer trial graphs will need to be made, but
# the resulting table is also larger. Increase this starting value
# if you're impatient. After 50 failures, c will be increased by 0.025.
if initC is None:
initC = 1.5
c = initC
if cIncrement is None:
cIncrement = 0.0025
if cTries is None:
cTries = 50
# Number of trial graphs so far
num_graphs = 0
sys.stderr.write('Generating graphs... ')
while 1:
# N is the number of vertices in the graph G
N = int(c*K)
num_graphs = num_graphs + 1
if (num_graphs % cTries) == 0:
# Enough failures at this multiplier,
# increase the multiplier and keep trying....
c = c + cIncrement
# Whats good with searching for a better
# hash function if we exceed the size
# of a function we've generated in the past....
if minC is not None and \
c > minC:
c = initC
sys.stderr.write(' -- c > minC, resetting c to %0.4f\n' % c)
else:
sys.stderr.write(' -- increasing c to %0.4f\n' % c)
sys.stderr.write('Generating graphs... ')
# Output a progress message
sys.stderr.write( str(num_graphs) + ' ')
sys.stderr.flush()
# Create graph w/ N vertices
G = Graph(N)
# Save the seeds used to generate
# the following two hash functions.
_seeds = whrandom._inst._seed
# Create 2 random hash functions
f1 = Hash(N, caseInsensitive)
f2 = Hash(N, caseInsensitive)
# Set the initial hash function seed values if passed in.
# Doing this protects our hash functions from
# changes to whrandom's behavior.
if f1Seed is not None:
f1.seed = f1Seed
f1Seed = None
fSpecifiedSeeds = 1
if f2Seed is not None:
f2.seed = f2Seed
f2Seed = None
fSpecifiedSeeds = 1
# Connect vertices given by the values of the two hash functions
# for each key. Associate the desired hash value with each
# edge.
for k, v in keys:
h1 = f1(k) ; h2 = f2(k)
G.connect( h1,h2, v)
# Check if the resulting graph is acyclic; if it is,
# we're done with step 1.
if G.is_acyclic():
break
elif fSpecifiedSeeds:
sys.stderr.write('\nThe initial f1/f2 seeds you specified didn\'t generate a perfect hash function: \n')
sys.stderr.write('f1 seed: %s\n' % f1.seed)
sys.stderr.write('f2 seed: %s\n' % f2.seed)
sys.stderr.write('multipler: %s\n' % c)
sys.stderr.write('Your data has likely changed, or you forgot what your initial multiplier should be.\n')
sys.stderr.write('continuing the search for a perfect hash function......\n')
fSpecifiedSeeds = 0
# Now we have an acyclic graph, so we assign values to each vertex
# such that, for each edge, you can add the values for the two vertices
# involved and get the desired value for that edge -- which is the
# desired hash key. This task is dead easy, because the graph is acyclic.
sys.stderr.write('\nAcyclic graph found; computing vertex values...\n')
G.assign_values()
sys.stderr.write('Checking uniqueness of hash values...\n')
# Sanity check the result by actually verifying that all the keys
# hash to the right value.
cchMaxKey = 0
maxHashValue = 0
for k, v in keys:
hash1 = G.values[ f1(k) ]
hash2 = G.values[ f2(k) ]
if hash1 > maxHashValue:
maxHashValue = hash1
if hash2 > maxHashValue:
maxHashValue = hash2
perfecthash = (hash1 + hash2) % N
assert perfecthash == v
cch = len(k)
if cch > cchMaxKey:
cchMaxKey = cch
sys.stderr.write('Found perfect hash function!\n')
sys.stderr.write('\nIn order to regenerate this hash function, \n')
sys.stderr.write('you need to pass these following values back in:\n')
sys.stderr.write('f1 seed: %s\n' % repr(f1.seed))
sys.stderr.write('f2 seed: %s\n' % repr(f2.seed))
sys.stderr.write('initial multipler: %s\n' % c)
return PerfectHash(cchMaxKey, f1, f2, G, N, len(keys), maxHashValue)
"""
static
PyObject *codec_tuple(PyObject *unicode,
int len)
{
PyObject *v,*w;
if (unicode == NULL)
return NULL;
v = PyTuple_New(2);
if (v == NULL) {
Py_DECREF(unicode);
return NULL;
}
PyTuple_SET_ITEM(v,0,unicode);
w = PyInt_FromLong(len);
if (w == NULL) {
Py_DECREF(v);
return NULL;
}
PyTuple_SET_ITEM(v,1,w);
return v;
}
static PyObject *
ucn_decode(PyObject *self,
PyObject *args)
{
const char *data;
int size;
const char *errors = NULL;
PyObject *mapping = NULL;
if (!PyArg_ParseTuple(args, "t#|z:ucn_decode",
&data, &size, &errors))
return NULL;
if (mapping == Py_None)
mapping = NULL;
return codec_tuple(PyUnicode_DecodeNamedUnicodeEscape(data, size, errors),
size);
}
static PyMethodDef _codecs_functions[] = {
{ "ucn_decode", ucn_decode, 1 },
};
DL_EXPORT(void)
init_ucn()
{
Py_InitModule("_ucn", _codecs_functions);
}
"""
Reference in New Issue View Git Blame Copy Permalink