bpo-24658: Fix read/write greater than 2 GiB on macOS (GH-1705)
On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
This commit is contained in:
parent
0f11a88622
commit
74a8b6ea7e
|
@ -81,6 +81,19 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx(
|
|||
#ifndef Py_LIMITED_API
|
||||
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
|
||||
|
||||
#if defined(MS_WINDOWS) || defined(__APPLE__)
|
||||
/* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
|
||||
On macOS 10.13, read() and write() with more than INT_MAX bytes
|
||||
fail with EINVAL (bpo-24658). */
|
||||
# define _PY_READ_MAX INT_MAX
|
||||
# define _PY_WRITE_MAX INT_MAX
|
||||
#else
|
||||
/* write() should truncate the input to PY_SSIZE_T_MAX bytes,
|
||||
but it's safer to do it ourself to have a portable behaviour */
|
||||
# define _PY_READ_MAX PY_SSIZE_T_MAX
|
||||
# define _PY_WRITE_MAX PY_SSIZE_T_MAX
|
||||
#endif
|
||||
|
||||
#ifdef MS_WINDOWS
|
||||
struct _Py_stat_struct {
|
||||
unsigned long st_dev;
|
||||
|
|
|
@ -5,12 +5,12 @@ import os
|
|||
import stat
|
||||
import sys
|
||||
import unittest
|
||||
from test.support import TESTFN, requires, unlink
|
||||
from test.support import TESTFN, requires, unlink, bigmemtest
|
||||
import io # C implementation of io
|
||||
import _pyio as pyio # Python implementation of io
|
||||
|
||||
# size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
|
||||
size = 2500000000
|
||||
size = 2_500_000_000
|
||||
|
||||
class LargeFileTest:
|
||||
"""Test that each file function works as expected for large
|
||||
|
@ -45,6 +45,15 @@ class LargeFileTest:
|
|||
raise cls.failureException('File was not truncated by opening '
|
||||
'with mode "wb"')
|
||||
|
||||
# _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
|
||||
# so memuse=2 is needed
|
||||
@bigmemtest(size=size, memuse=2, dry_run=False)
|
||||
def test_large_read(self, _size):
|
||||
# bpo-24658: Test that a read greater than 2GB does not fail.
|
||||
with self.open(TESTFN, "rb") as f:
|
||||
self.assertEqual(len(f.read()), size + 1)
|
||||
self.assertEqual(f.tell(), size + 1)
|
||||
|
||||
def test_osstat(self):
|
||||
self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
|
||||
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
|
|
@ -791,11 +791,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size)
|
|||
if (size < 0)
|
||||
return _io_FileIO_readall_impl(self);
|
||||
|
||||
#ifdef MS_WINDOWS
|
||||
/* On Windows, the count parameter of read() is an int */
|
||||
if (size > INT_MAX)
|
||||
size = INT_MAX;
|
||||
#endif
|
||||
if (size > _PY_READ_MAX) {
|
||||
size = _PY_READ_MAX;
|
||||
}
|
||||
|
||||
bytes = PyBytes_FromStringAndSize(NULL, size);
|
||||
if (bytes == NULL)
|
||||
|
|
|
@ -1471,18 +1471,9 @@ _Py_read(int fd, void *buf, size_t count)
|
|||
* handler raised an exception. */
|
||||
assert(!PyErr_Occurred());
|
||||
|
||||
#ifdef MS_WINDOWS
|
||||
if (count > INT_MAX) {
|
||||
/* On Windows, the count parameter of read() is an int */
|
||||
count = INT_MAX;
|
||||
if (count > _PY_READ_MAX) {
|
||||
count = _PY_READ_MAX;
|
||||
}
|
||||
#else
|
||||
if (count > PY_SSIZE_T_MAX) {
|
||||
/* if count is greater than PY_SSIZE_T_MAX,
|
||||
* read() result is undefined */
|
||||
count = PY_SSIZE_T_MAX;
|
||||
}
|
||||
#endif
|
||||
|
||||
_Py_BEGIN_SUPPRESS_IPH
|
||||
do {
|
||||
|
@ -1533,15 +1524,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
|
|||
depending on heap usage). */
|
||||
count = 32767;
|
||||
}
|
||||
else if (count > INT_MAX)
|
||||
count = INT_MAX;
|
||||
#else
|
||||
if (count > PY_SSIZE_T_MAX) {
|
||||
/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer
|
||||
* to do it ourself to have a portable behaviour. */
|
||||
count = PY_SSIZE_T_MAX;
|
||||
}
|
||||
#endif
|
||||
if (count > _PY_WRITE_MAX) {
|
||||
count = _PY_WRITE_MAX;
|
||||
}
|
||||
|
||||
if (gil_held) {
|
||||
do {
|
||||
|
|
Loading…
Reference in New Issue