mirror of https://github.com/python/cpython
This is my patch:
[ 1180995 ] binary formats for marshalling floats Adds 2 new type codes for marshal (binary floats and binary complexes), a new marshal version (2), updates MAGIC and fiddles the de-serializing of code objects to be less likely to clobber the real reason for failing if it fails.
This commit is contained in:
Michael W. Hudson
parent
268e61cf74
commit
df88846ebc
|
|
@ -7,7 +7,7 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define Py_MARSHAL_VERSION 1
|
||||
#define Py_MARSHAL_VERSION 2
|
||||
|
||||
PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
|
||||
PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);
|
||||
|
|
|
|||
|
|
@ -73,20 +73,34 @@ class FloatTestCase(unittest.TestCase):
|
|||
n /= 123.4567
|
||||
|
||||
f = 0.0
|
||||
s = marshal.dumps(f)
|
||||
s = marshal.dumps(f, 2)
|
||||
got = marshal.loads(s)
|
||||
self.assertEqual(f, got)
|
||||
# and with version <= 1 (floats marshalled differently then)
|
||||
s = marshal.dumps(f, 1)
|
||||
got = marshal.loads(s)
|
||||
self.assertEqual(f, got)
|
||||
|
||||
n = sys.maxint * 3.7e-250
|
||||
while n < small:
|
||||
for expected in (-n, n):
|
||||
f = float(expected)
|
||||
|
||||
s = marshal.dumps(f)
|
||||
got = marshal.loads(s)
|
||||
self.assertEqual(f, got)
|
||||
|
||||
s = marshal.dumps(f, 1)
|
||||
got = marshal.loads(s)
|
||||
self.assertEqual(f, got)
|
||||
|
||||
marshal.dump(f, file(test_support.TESTFN, "wb"))
|
||||
got = marshal.load(file(test_support.TESTFN, "rb"))
|
||||
self.assertEqual(f, got)
|
||||
|
||||
marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
|
||||
got = marshal.load(file(test_support.TESTFN, "rb"))
|
||||
self.assertEqual(f, got)
|
||||
n *= 123.4567
|
||||
os.unlink(test_support.TESTFN)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ What's New in Python 2.5 alpha 1?
|
|||
Core and builtins
|
||||
-----------------
|
||||
|
||||
- SF patch #1180995: marshal now uses a binary format by default when
|
||||
serializing floats.
|
||||
|
||||
- SF patch #1181301: on platforms that appear to use IEEE 754 floats,
|
||||
the routines that promise to produce IEEE 754 binary representations
|
||||
of floats now simply copy bytes around.
|
||||
|
|
|
|||
|
|
@ -50,8 +50,9 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
|
|||
Python 2.4a0: 62041
|
||||
Python 2.4a3: 62051
|
||||
Python 2.4b1: 62061
|
||||
Python 2.5a0: 62071
|
||||
*/
|
||||
#define MAGIC (62061 | ((long)'\r'<<16) | ((long)'\n'<<24))
|
||||
#define MAGIC (62071 | ((long)'\r'<<16) | ((long)'\n'<<24))
|
||||
|
||||
/* Magic word as global; note that _PyImport_Init() can change the
|
||||
value of this global to accommodate for alterations of how the
|
||||
|
|
|
|||
238
Python/marshal.c
238
Python/marshal.c
|
|
@ -15,28 +15,30 @@
|
|||
*/
|
||||
#define MAX_MARSHAL_STACK_DEPTH 5000
|
||||
|
||||
#define TYPE_NULL '0'
|
||||
#define TYPE_NONE 'N'
|
||||
#define TYPE_FALSE 'F'
|
||||
#define TYPE_TRUE 'T'
|
||||
#define TYPE_STOPITER 'S'
|
||||
#define TYPE_ELLIPSIS '.'
|
||||
#define TYPE_INT 'i'
|
||||
#define TYPE_INT64 'I'
|
||||
#define TYPE_FLOAT 'f'
|
||||
#define TYPE_COMPLEX 'x'
|
||||
#define TYPE_LONG 'l'
|
||||
#define TYPE_STRING 's'
|
||||
#define TYPE_INTERNED 't'
|
||||
#define TYPE_STRINGREF 'R'
|
||||
#define TYPE_TUPLE '('
|
||||
#define TYPE_LIST '['
|
||||
#define TYPE_DICT '{'
|
||||
#define TYPE_CODE 'c'
|
||||
#define TYPE_UNICODE 'u'
|
||||
#define TYPE_UNKNOWN '?'
|
||||
#define TYPE_SET '<'
|
||||
#define TYPE_FROZENSET '>'
|
||||
#define TYPE_NULL '0'
|
||||
#define TYPE_NONE 'N'
|
||||
#define TYPE_FALSE 'F'
|
||||
#define TYPE_TRUE 'T'
|
||||
#define TYPE_STOPITER 'S'
|
||||
#define TYPE_ELLIPSIS '.'
|
||||
#define TYPE_INT 'i'
|
||||
#define TYPE_INT64 'I'
|
||||
#define TYPE_FLOAT 'f'
|
||||
#define TYPE_BINARY_FLOAT 'g'
|
||||
#define TYPE_COMPLEX 'x'
|
||||
#define TYPE_BINARY_COMPLEX 'y'
|
||||
#define TYPE_LONG 'l'
|
||||
#define TYPE_STRING 's'
|
||||
#define TYPE_INTERNED 't'
|
||||
#define TYPE_STRINGREF 'R'
|
||||
#define TYPE_TUPLE '('
|
||||
#define TYPE_LIST '['
|
||||
#define TYPE_DICT '{'
|
||||
#define TYPE_CODE 'c'
|
||||
#define TYPE_UNICODE 'u'
|
||||
#define TYPE_UNKNOWN '?'
|
||||
#define TYPE_SET '<'
|
||||
#define TYPE_FROZENSET '>'
|
||||
|
||||
typedef struct {
|
||||
FILE *fp;
|
||||
|
|
@ -47,6 +49,7 @@ typedef struct {
|
|||
char *ptr;
|
||||
char *end;
|
||||
PyObject *strings; /* dict on marshal, list on unmarshal */
|
||||
int version;
|
||||
} WFILE;
|
||||
|
||||
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
|
||||
|
|
@ -165,32 +168,62 @@ w_object(PyObject *v, WFILE *p)
|
|||
w_short(ob->ob_digit[i], p);
|
||||
}
|
||||
else if (PyFloat_Check(v)) {
|
||||
char buf[256]; /* Plenty to format any double */
|
||||
PyFloat_AsReprString(buf, (PyFloatObject *)v);
|
||||
n = strlen(buf);
|
||||
w_byte(TYPE_FLOAT, p);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
if (p->version > 1) {
|
||||
char buf[8];
|
||||
if (_PyFloat_Pack8(PyFloat_AsDouble(v),
|
||||
buf, 1) < 0) {
|
||||
p->error = 1;
|
||||
return;
|
||||
}
|
||||
w_byte(TYPE_BINARY_FLOAT, p);
|
||||
w_string(buf, 8, p);
|
||||
}
|
||||
else {
|
||||
char buf[256]; /* Plenty to format any double */
|
||||
PyFloat_AsReprString(buf, (PyFloatObject *)v);
|
||||
n = strlen(buf);
|
||||
w_byte(TYPE_FLOAT, p);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
}
|
||||
}
|
||||
#ifndef WITHOUT_COMPLEX
|
||||
else if (PyComplex_Check(v)) {
|
||||
char buf[256]; /* Plenty to format any double */
|
||||
PyFloatObject *temp;
|
||||
w_byte(TYPE_COMPLEX, p);
|
||||
temp = (PyFloatObject*)PyFloat_FromDouble(
|
||||
PyComplex_RealAsDouble(v));
|
||||
PyFloat_AsReprString(buf, temp);
|
||||
Py_DECREF(temp);
|
||||
n = strlen(buf);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
temp = (PyFloatObject*)PyFloat_FromDouble(
|
||||
PyComplex_ImagAsDouble(v));
|
||||
PyFloat_AsReprString(buf, temp);
|
||||
Py_DECREF(temp);
|
||||
n = strlen(buf);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
if (p->version > 1) {
|
||||
char buf[8];
|
||||
if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
|
||||
buf, 1) < 0) {
|
||||
p->error = 1;
|
||||
return;
|
||||
}
|
||||
w_byte(TYPE_BINARY_COMPLEX, p);
|
||||
w_string(buf, 8, p);
|
||||
if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
|
||||
buf, 1) < 0) {
|
||||
p->error = 1;
|
||||
return;
|
||||
}
|
||||
w_string(buf, 8, p);
|
||||
}
|
||||
else {
|
||||
char buf[256]; /* Plenty to format any double */
|
||||
PyFloatObject *temp;
|
||||
w_byte(TYPE_COMPLEX, p);
|
||||
temp = (PyFloatObject*)PyFloat_FromDouble(
|
||||
PyComplex_RealAsDouble(v));
|
||||
PyFloat_AsReprString(buf, temp);
|
||||
Py_DECREF(temp);
|
||||
n = strlen(buf);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
temp = (PyFloatObject*)PyFloat_FromDouble(
|
||||
PyComplex_ImagAsDouble(v));
|
||||
PyFloat_AsReprString(buf, temp);
|
||||
Py_DECREF(temp);
|
||||
n = strlen(buf);
|
||||
w_byte(n, p);
|
||||
w_string(buf, n, p);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
else if (PyString_Check(v)) {
|
||||
|
|
@ -335,6 +368,7 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
|
|||
wf.error = 0;
|
||||
wf.depth = 0;
|
||||
wf.strings = NULL;
|
||||
wf.version = version;
|
||||
w_long(x, &wf);
|
||||
}
|
||||
|
||||
|
|
@ -346,6 +380,7 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
|
|||
wf.error = 0;
|
||||
wf.depth = 0;
|
||||
wf.strings = (version > 0) ? PyDict_New() : NULL;
|
||||
wf.version = version;
|
||||
w_object(x, &wf);
|
||||
Py_XDECREF(wf.strings);
|
||||
}
|
||||
|
|
@ -519,6 +554,22 @@ r_object(RFILE *p)
|
|||
return PyFloat_FromDouble(dx);
|
||||
}
|
||||
|
||||
case TYPE_BINARY_FLOAT:
|
||||
{
|
||||
char buf[8];
|
||||
double x;
|
||||
if (r_string(buf, 8, p) != 8) {
|
||||
PyErr_SetString(PyExc_EOFError,
|
||||
"EOF read where object expected");
|
||||
return NULL;
|
||||
}
|
||||
x = _PyFloat_Unpack8(buf, 1);
|
||||
if (x == -1.0 && PyErr_Occurred()) {
|
||||
return NULL;
|
||||
}
|
||||
return PyFloat_FromDouble(x);
|
||||
}
|
||||
|
||||
#ifndef WITHOUT_COMPLEX
|
||||
case TYPE_COMPLEX:
|
||||
{
|
||||
|
|
@ -546,6 +597,31 @@ r_object(RFILE *p)
|
|||
PyFPE_END_PROTECT(c)
|
||||
return PyComplex_FromCComplex(c);
|
||||
}
|
||||
|
||||
case TYPE_BINARY_COMPLEX:
|
||||
{
|
||||
char buf[8];
|
||||
Py_complex c;
|
||||
if (r_string(buf, 8, p) != 8) {
|
||||
PyErr_SetString(PyExc_EOFError,
|
||||
"EOF read where object expected");
|
||||
return NULL;
|
||||
}
|
||||
c.real = _PyFloat_Unpack8(buf, 1);
|
||||
if (c.real == -1.0 && PyErr_Occurred()) {
|
||||
return NULL;
|
||||
}
|
||||
if (r_string(buf, 8, p) != 8) {
|
||||
PyErr_SetString(PyExc_EOFError,
|
||||
"EOF read where object expected");
|
||||
return NULL;
|
||||
}
|
||||
c.imag = _PyFloat_Unpack8(buf, 1);
|
||||
if (c.imag == -1.0 && PyErr_Occurred()) {
|
||||
return NULL;
|
||||
}
|
||||
return PyComplex_FromCComplex(c);
|
||||
}
|
||||
#endif
|
||||
|
||||
case TYPE_INTERNED:
|
||||
|
|
@ -707,30 +783,63 @@ r_object(RFILE *p)
|
|||
return NULL;
|
||||
}
|
||||
else {
|
||||
int argcount = r_long(p);
|
||||
int nlocals = r_long(p);
|
||||
int stacksize = r_long(p);
|
||||
int flags = r_long(p);
|
||||
PyObject *code = r_object(p);
|
||||
PyObject *consts = r_object(p);
|
||||
PyObject *names = r_object(p);
|
||||
PyObject *varnames = r_object(p);
|
||||
PyObject *freevars = r_object(p);
|
||||
PyObject *cellvars = r_object(p);
|
||||
PyObject *filename = r_object(p);
|
||||
PyObject *name = r_object(p);
|
||||
int firstlineno = r_long(p);
|
||||
PyObject *lnotab = r_object(p);
|
||||
int argcount;
|
||||
int nlocals;
|
||||
int stacksize;
|
||||
int flags;
|
||||
PyObject *code = NULL;
|
||||
PyObject *consts = NULL;
|
||||
PyObject *names = NULL;
|
||||
PyObject *varnames = NULL;
|
||||
PyObject *freevars = NULL;
|
||||
PyObject *cellvars = NULL;
|
||||
PyObject *filename = NULL;
|
||||
PyObject *name = NULL;
|
||||
int firstlineno;
|
||||
PyObject *lnotab = NULL;
|
||||
|
||||
v = NULL;
|
||||
|
||||
if (!PyErr_Occurred()) {
|
||||
v = (PyObject *) PyCode_New(
|
||||
argcount = r_long(p);
|
||||
nlocals = r_long(p);
|
||||
stacksize = r_long(p);
|
||||
flags = r_long(p);
|
||||
code = r_object(p);
|
||||
if (code == NULL)
|
||||
goto code_error;
|
||||
consts = r_object(p);
|
||||
if (consts == NULL)
|
||||
goto code_error;
|
||||
names = r_object(p);
|
||||
if (names == NULL)
|
||||
goto code_error;
|
||||
varnames = r_object(p);
|
||||
if (varnames == NULL)
|
||||
goto code_error;
|
||||
freevars = r_object(p);
|
||||
if (freevars == NULL)
|
||||
goto code_error;
|
||||
cellvars = r_object(p);
|
||||
if (cellvars == NULL)
|
||||
goto code_error;
|
||||
filename = r_object(p);
|
||||
if (filename == NULL)
|
||||
goto code_error;
|
||||
name = r_object(p);
|
||||
if (name == NULL)
|
||||
goto code_error;
|
||||
firstlineno = r_long(p);
|
||||
lnotab = r_object(p);
|
||||
if (lnotab == NULL)
|
||||
goto code_error;
|
||||
|
||||
v = (PyObject *) PyCode_New(
|
||||
argcount, nlocals, stacksize, flags,
|
||||
code, consts, names, varnames,
|
||||
freevars, cellvars, filename, name,
|
||||
firstlineno, lnotab);
|
||||
}
|
||||
else
|
||||
v = NULL;
|
||||
|
||||
code_error:
|
||||
Py_XDECREF(code);
|
||||
Py_XDECREF(consts);
|
||||
Py_XDECREF(names);
|
||||
|
|
@ -882,6 +991,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
|
|||
wf.end = wf.ptr + PyString_Size(wf.str);
|
||||
wf.error = 0;
|
||||
wf.depth = 0;
|
||||
wf.version = version;
|
||||
wf.strings = (version > 0) ? PyDict_New() : NULL;
|
||||
w_object(x, &wf);
|
||||
Py_XDECREF(wf.strings);
|
||||
|
|
|
|||
Loading…
Reference in New Issue