traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New version by Digital Creations, supports binary format compatible 

with cPickle.
This commit is contained in:
Guido van Rossum 1997-04-09 17:32:51 +00:00
parent de65527e4b
commit b72cf2d697
1 changed files with 749 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

604
Lib/pickle.py
View File

@ -1,3 +1,56 @@
# $Id$
#
# Copyright
#
# Copyright 1996 Digital Creations, L.C., 910 Princess Anne
# Street, Suite 300, Fredericksburg, Virginia 22401 U.S.A. All
# rights reserved. Copyright in this software is owned by DCLC,
# unless otherwise indicated. Permission to use, copy and
# distribute this software is hereby granted, provided that the
# above copyright notice appear in all copies and that both that
# copyright notice and this permission notice appear. Note that
# any product, process or technology described in this software
# may be the subject of other Intellectual Property rights
# reserved by Digital Creations, L.C. and are not licensed
# hereunder.
#
# Trademarks
#
# Digital Creations & DCLC, are trademarks of Digital Creations, L.C..
# All other trademarks are owned by their respective companies.
#
# No Warranty
#
# The software is provided "as is" without warranty of any kind,
# either express or implied, including, but not limited to, the
# implied warranties of merchantability, fitness for a particular
# purpose, or non-infringement. This software could include
# technical inaccuracies or typographical errors. Changes are
# periodically made to the software; these changes will be
# incorporated in new editions of the software. DCLC may make
# improvements and/or changes in this software at any time
# without notice.
#
# Limitation Of Liability
#
# In no event will DCLC be liable for direct, indirect, special,
# incidental, economic, cover, or consequential damages arising
# out of the use of or inability to use this software even if
# advised of the possibility of such damages. Some states do not
# allow the exclusion or limitation of implied warranties or
# limitation of liability for incidental or consequential
# damages, so the above limitation or exclusion may not apply to
# you.
#
#
# If you have questions regarding this software,
# contact:
#
# Jim Fulton, jim@digicool.com
#
# (540) 371-6909
#
"""\
Pickling Algorithm
------------------
@ -128,90 +181,197 @@ the old value, not the modified one. (XXX There are two problems here:
I have no answers. Garbage Collection may also become a problem here.)
"""
__version__ = "1.6" # Code version
__version__ = "1.7" # Code version
from types import *
import string
from copy_reg import *
import string, marshal
format_version = "1.1" # File format version we write
compatible_formats = ["1.0"] # Old format versions we can read
format_version = "1.2" # File format version we write
compatible_formats = ["1.0", "1.1"] # Old format versions we can read
mdumps = marshal.dumps
mloads = marshal.loads
PicklingError = "pickle.PicklingError"
AtomicTypes = [NoneType, IntType, FloatType, StringType]
def safe(object):
t = type(object)
if t in AtomicTypes:
return 1
if t is TupleType:
for item in object:
if not safe(item): return 0
return 1
return 0
UnpicklingError = "pickle.UnpicklingError"
MARK = '('
POP = '0'
DUP = '2'
STOP = '.'
TUPLE = 't'
LIST = 'l'
DICT = 'd'
INST = 'i'
CLASS = 'c'
GET = 'g'
PUT = 'p'
APPEND = 'a'
SETITEM = 's'
BUILD = 'b'
NONE = 'N'
INT = 'I'
LONG = 'L'
POP = '0'
POP_MARK = '1'
DUP = '2'
FLOAT = 'F'
STRING = 'S'
INT = 'I'
BININT = 'J'
BININT1 = 'K'
LONG = 'L'
BININT2 = 'M'
NONE = 'N'
PERSID = 'P'
AtomicKeys = [NONE, INT, LONG, FLOAT, STRING]
AtomicMap = {
NoneType: NONE,
IntType: INT,
LongType: LONG,
FloatType: FLOAT,
StringType: STRING,
}
BINPERSID = 'Q'
REDUCE = 'R'
STRING = 'S'
BINSTRING = 'T'
SHORT_BINSTRING = 'U'
APPEND = 'a'
BUILD = 'b'
GLOBAL = 'c'
DICT = 'd'
EMPTY_DICT = '}'
APPENDS = 'e'
GET = 'g'
BINGET = 'h'
INST = 'i'
LONG_BINGET = 'j'
LIST = 'l'
EMPTY_LIST = ']'
OBJ = 'o'
PUT = 'p'
BINPUT = 'q'
LONG_BINPUT = 'r'
SETITEM = 's'
TUPLE = 't'
EMPTY_TUPLE = ')'
SETITEMS = 'u'
class Pickler:
def __init__(self, file):
def __init__(self, file, bin = 0):
self.write = file.write
self.memo = {}
self.bin = bin
def dump(self, object):
self.save(object)
self.write(STOP)
def save(self, object):
def dump_special(self, callable, args, state = None):
if (type(args) is not TupleType):
raise PicklingError, "Second argument to dump_special " \
"must be a tuple"
self.save_reduce(callable, args, state)
self.write(STOP)
def put(self, i):
if (self.bin):
s = mdumps(i)[1:]
if (i < 256):
return BINPUT + s[0]
return LONG_BINPUT + s
return PUT + `i` + '\n'
def get(self, i):
if (self.bin):
s = mdumps(i)[1:]
if (i < 256):
return BINGET + s[0]
return LONG_BINGET + s
return GET + `i` + '\n'
def save(self, object, pers_save = 0):
memo = self.memo
if (not pers_save):
pid = self.persistent_id(object)
if pid:
self.write(PERSID + str(pid) + '\n')
if (pid is not None):
self.save_pers(pid)
return
d = id(object)
if self.memo.has_key(d):
self.write(GET + `d` + '\n')
return
t = type(object)
if ((t is TupleType) and (len(object) == 0)):
if (self.bin):
self.save_empty_tuple(object)
else:
self.save_tuple(object)
return
if memo.has_key(d):
self.write(self.get(memo[d][0]))
return
try:
f = self.dispatch[t]
except KeyError:
if hasattr(object, '__class__'):
f = self.dispatch[InstanceType]
else:
pid = self.inst_persistent_id(object)
if pid is not None:
self.save_pers(pid)
return
try:
reduce = dispatch_table[t]
except KeyError:
try:
reduce = object.__reduce__
except AttributeError:
raise PicklingError, \
"can't pickle %s objects" % `t.__name__`
else:
tup = reduce()
else:
tup = reduce(object)
if (type(tup) is not TupleType):
raise PicklingError, "Value returned by %s must be a " \
"tuple" % reduce
l = len(tup)
if ((l != 2) and (l != 3)):
raise PicklingError, "tuple returned by %s must contain " \
"only two or three elements" % reduce
callable = tup[0]
arg_tup = tup[1]
if (l > 2):
state = tup[2]
else:
state = None
if (type(arg_tup) is not TupleType):
raise PicklingError, "Second element of tuple returned " \
"by %s must be a tuple" % reduce
self.save_reduce(callable, arg_tup, state)
return
f(self, object)
def persistent_id(self, object):
return None
def inst_persistent_id(self, object):
return None
def save_pers(self, pid):
if (not self.bin):
self.write(PERSID + str(pid) + '\n')
else:
self.save(pid, 1)
self.write(BINPERSID)
def save_reduce(self, callable, arg_tup, state = None):
write = self.write
save = self.save
save(callable)
save(arg_tup)
write(REDUCE)
if (state is not None):
save(state)
write(BUILD)
dispatch = {}
def save_none(self, object):
@ -219,6 +379,18 @@ class Pickler:
dispatch[NoneType] = save_none
def save_int(self, object):
if (self.bin):
i = mdumps(object)[1:]
if (i[-2:] == '\000\000'):
if (i[-3] == '\000'):
self.write(BININT1 + i[:-3])
return
self.write(BININT2 + i[:-2])
return
self.write(BININT + i)
else:
self.write(INT + `object` + '\n')
dispatch[IntType] = save_int
@ -232,95 +404,150 @@ class Pickler:
def save_string(self, object):
d = id(object)
memo = self.memo
if (self.bin):
l = len(object)
s = mdumps(l)[1:]
if (l < 256):
self.write(SHORT_BINSTRING + s[0] + object)
else:
self.write(BINSTRING + s + object)
else:
self.write(STRING + `object` + '\n')
self.write(PUT + `d` + '\n')
self.memo[d] = object
memo_len = len(memo)
self.write(self.put(memo_len))
memo[d] = (memo_len, object)
dispatch[StringType] = save_string
def save_tuple(self, object):
d = id(object)
write = self.write
save = self.save
has_key = self.memo.has_key
memo = self.memo
d = id(object)
write(MARK)
n = len(object)
for k in range(n):
save(object[k])
if has_key(d):
# Saving object[k] has saved us!
while k >= 0:
write(POP)
k = k-1
write(GET + `d` + '\n')
break
else:
write(TUPLE + PUT + `d` + '\n')
self.memo[d] = object
for element in object:
save(element)
if (len(object) and memo.has_key(d)):
if (self.bin):
write(POP_MARK + self.get(memo[d][0]))
return
write(POP * (len(object) + 1) + self.get(mem[d][0]))
return
memo_len = len(memo)
self.write(TUPLE + self.put(memo_len))
memo[d] = (memo_len, object)
dispatch[TupleType] = save_tuple
def save_empty_tuple(self, object):
self.write(EMPTY_TUPLE)
def save_list(self, object):
d = id(object)
write = self.write
save = self.save
write(MARK)
n = len(object)
for k in range(n):
item = object[k]
if not safe(item):
break
save(item)
memo = self.memo
if (self.bin):
write(EMPTY_LIST)
else:
k = n
write(LIST + PUT + `d` + '\n')
self.memo[d] = object
for k in range(k, n):
item = object[k]
save(item)
write(MARK + LIST)
memo_len = len(memo)
write(self.put(memo_len))
memo[d] = (memo_len, object)
using_appends = (self.bin and (len(object) > 1))
if (using_appends):
write(MARK)
for element in object:
save(element)
if (not using_appends):
write(APPEND)
if (using_appends):
write(APPENDS)
dispatch[ListType] = save_list
def save_dict(self, object):
d = id(object)
write = self.write
save = self.save
write(MARK)
items = object.items()
n = len(items)
for k in range(n):
key, value = items[k]
if not safe(key) or not safe(value):
break
save(key)
save(value)
memo = self.memo
if (self.bin):
write(EMPTY_DICT)
else:
k = n
self.write(DICT + PUT + `d` + '\n')
self.memo[d] = object
for k in range(k, n):
key, value = items[k]
write(MARK + DICT)
memo_len = len(memo)
self.write(self.put(memo_len))
memo[d] = (memo_len, object)
using_setitems = (self.bin and (len(object) > 1))
if (using_setitems):
write(MARK)
items = object.items()
for key, value in items:
save(key)
save(value)
if (not using_setitems):
write(SETITEM)
if (using_setitems):
write(SETITEMS)
dispatch[DictionaryType] = save_dict
def save_inst(self, object):
d = id(object)
cls = object.__class__
memo = self.memo
write = self.write
save = self.save
module = whichmodule(cls)
name = cls.__name__
if hasattr(object, '__getinitargs__'):
args = object.__getinitargs__()
len(args) # XXX Assert it's a sequence
else:
args = ()
write(MARK)
if (self.bin):
save(cls)
for arg in args:
save(arg)
memo_len = len(memo)
if (self.bin):
write(OBJ + self.put(memo_len))
else:
module = whichmodule(cls, cls.__name__)
name = cls.__name__
write(INST + module + '\n' + name + '\n' +
PUT + `d` + '\n')
self.memo[d] = object
self.put(memo_len))
memo[d] = (memo_len, object)
try:
getstate = object.__getstate__
except AttributeError:
@ -331,18 +558,27 @@ class Pickler:
write(BUILD)
dispatch[InstanceType] = save_inst
def save_class(self, object):
d = id(object)
module = whichmodule(object)
def save_global(self, object, name = None):
write = self.write
memo = self.memo
if (name is None):
name = object.__name__
self.write(CLASS + module + '\n' + name + '\n' +
PUT + `d` + '\n')
dispatch[ClassType] = save_class
module = whichmodule(object, name)
memo_len = len(memo)
write(GLOBAL + module + '\n' + name + '\n' +
self.put(memo_len))
memo[id(object)] = (memo_len, object)
dispatch[ClassType] = save_global
dispatch[FunctionType] = save_global
dispatch[BuiltinFunctionType] = save_global
classmap = {}
def whichmodule(cls):
def whichmodule(cls, clsname):
"""Figure out the module in which a class occurs.
Search sys.modules for the module.
@ -353,10 +589,9 @@ def whichmodule(cls):
if classmap.has_key(cls):
return classmap[cls]
import sys
clsname = cls.__name__
for name, module in sys.modules.items():
if name != '__main__' and \
hasattr(module, clsname) and \
if hasattr(module, clsname) and \
getattr(module, clsname) is cls:
break
else:
@ -403,6 +638,15 @@ class Unpickler:
self.append(self.persistent_load(pid))
dispatch[PERSID] = load_persid
def load_binpersid(self):
stack = self.stack
pid = stack[-1]
del stack[-1]
self.append(self.persistent_load(pid))
dispatch[BINPERSID] = load_binpersid
def load_none(self):
self.append(None)
dispatch[NONE] = load_none
@ -411,6 +655,18 @@ class Unpickler:
self.append(string.atoi(self.readline()[:-1], 0))
dispatch[INT] = load_int
def load_binint(self):
self.append(mloads('i' + self.read(4)))
dispatch[BININT] = load_binint
def load_binint1(self):
self.append(mloads('i' + self.read(1) + '\000\000\000'))
dispatch[BININT1] = load_binint1
def load_binint2(self):
self.append(mloads('i' + self.read(2) + '\000\000'))
dispatch[BININT2] = load_binint2
def load_long(self):
self.append(string.atol(self.readline()[:-1], 0))
dispatch[LONG] = load_long
@ -424,11 +680,33 @@ class Unpickler:
{'__builtins__': {}})) # Let's be careful
dispatch[STRING] = load_string
def load_binstring(self):
len = mloads('i' + self.read(4))
self.append(self.read(len))
dispatch[BINSTRING] = load_binstring
def load_short_binstring(self):
len = mloads('i' + self.read(1) + '\000\000\000')
self.append(self.read(len))
dispatch[SHORT_BINSTRING] = load_short_binstring
def load_tuple(self):
k = self.marker()
self.stack[k:] = [tuple(self.stack[k+1:])]
dispatch[TUPLE] = load_tuple
def load_empty_tuple(self):
self.stack.append(())
dispatch[EMPTY_TUPLE] = load_empty_tuple
def load_empty_list(self):
self.stack.append([])
dispatch[EMPTY_LIST] = load_empty_list
def load_empty_dictionary(self):
self.stack.append({})
dispatch[EMPTY_DICT] = load_empty_dictionary
def load_list(self):
k = self.marker()
self.stack[k:] = [self.stack[k+1:]]
@ -452,35 +730,75 @@ class Unpickler:
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = self.find_class(module, name)
if (type(klass) is not ClassType):
raise SystemError, "Imported object %s from module %s is " \
"not a class" % (name, module)
value = apply(klass, args)
self.append(value)
dispatch[INST] = load_inst
def load_class(self):
def load_obj(self):
stack = self.stack
k = self.marker()
klass = stack[k + 1]
del stack[k + 1]
args = tuple(stack[k + 1:])
del stack[k:]
value = apply(klass, args)
self.append(value)
dispatch[OBJ] = load_obj
def load_global(self):
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = self.find_class(module, name)
self.append(klass)
return klass
dispatch[CLASS] = load_class
dispatch[GLOBAL] = load_global
def find_class(self, module, name):
env = {}
try:
klass = getattr(__import__(module), name)
except (ImportError, AttributeError):
exec 'from %s import %s' % (module, name) in env
except ImportError:
raise SystemError, \
"Failed to import class %s from module %s" % \
(name, module)
if type(klass) is BuiltinFunctionType:
raise SystemError, \
"Imported object %s from module %s is not a class" % \
(name, module)
klass = env[name]
return klass
def load_reduce(self):
stack = self.stack
callable = stack[-2]
arg_tup = stack[-1]
del stack[-2:]
if (type(callable) is not ClassType):
if (not safe_constructors.has_key(callable)):
try:
safe = callable.__safe_for_unpickling__
except AttributeError:
safe = None
if (not safe):
raise UnpicklingError, "%s is not safe for " \
"unpickling" % callable
value = apply(callable, arg_tup)
self.append(value)
dispatch[REDUCE] = load_reduce
def load_pop(self):
del self.stack[-1]
dispatch[POP] = load_pop
def load_pop_mark(self):
k = self.marker()
del self.stack[k:]
dispatch[POP_MARK] = load_pop_mark
def load_dup(self):
self.append(stack[-1])
dispatch[DUP] = load_dup
@ -489,10 +807,30 @@ class Unpickler:
self.append(self.memo[self.readline()[:-1]])
dispatch[GET] = load_get
def load_binget(self):
i = mloads('i' + self.read(1) + '\000\000\000')
self.append(self.memo[`i`])
dispatch[BINGET] = load_binget
def load_long_binget(self):
i = mloads('i' + self.read(4))
self.append(self.memo[`i`])
dispatch[LONG_BINGET] = load_long_binget
def load_put(self):
self.memo[self.readline()[:-1]] = self.stack[-1]
dispatch[PUT] = load_put
def load_binput(self):
i = mloads('i' + self.read(1) + '\000\000\000')
self.memo[`i`] = self.stack[-1]
dispatch[BINPUT] = load_binput
def load_long_binput(self):
i = mloads('i' + self.read(4))
self.memo[`i`] = self.stack[-1]
dispatch[LONG_BINPUT] = load_long_binput
def load_append(self):
stack = self.stack
value = stack[-1]
@ -501,6 +839,16 @@ class Unpickler:
list.append(value)
dispatch[APPEND] = load_append
def load_appends(self):
stack = self.stack
mark = self.marker()
list = stack[mark - 1]
for i in range(mark + 1, len(stack)):
list.append(stack[i])
del stack[mark:]
dispatch[APPENDS] = load_appends
def load_setitem(self):
stack = self.stack
value = stack[-1]
@ -510,6 +858,16 @@ class Unpickler:
dict[key] = value
dispatch[SETITEM] = load_setitem
def load_setitems(self):
stack = self.stack
mark = self.marker()
dict = stack[mark - 1]
for i in range(mark + 1, len(stack), 2):
dict[stack[i]] = stack[i + 1]
del stack[mark:]
dispatch[SETITEMS] = load_setitems
def load_build(self):
stack = self.stack
value = stack[-1]
@ -539,12 +897,12 @@ class Unpickler:
from StringIO import StringIO
def dump(object, file):
Pickler(file).dump(object)
def dump(object, file, bin = 0):
Pickler(file, bin).dump(object)
def dumps(object):
def dumps(object, bin = 0):
file = StringIO()
Pickler(file).dump(object)
Pickler(file, bin).dump(object)
return file.getvalue()
def load(file):
@ -562,10 +920,10 @@ class C:
return cmp(self.__dict__, other.__dict__)
def test():
fn = 'pickle_tmp'
fn = 'out'
c = C()
c.foo = 1
c.bar = 2L
c.bar = 2
x = [0, 1, 2, 3]
y = ('abc', 'abc', c, c)
x.append(y)