bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)

...and avoid using it on Solaris as it can raise EINVAL if offset is equal or bigger than the size of the file
This commit is contained in:
Giampaolo Rodola 2019-05-30 14:05:41 +08:00 committed by GitHub
parent a16387ab2d
commit 413d955f8e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 12 additions and 13 deletions

View File

@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``".
On macOS `fcopyfile`_ is used to copy the file content (not metadata).
On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports
copies between 2 regular file descriptors :func:`os.sendfile` is used.
On Linux :func:`os.sendfile` is used.
On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB
instead of 64 KiB) and a :func:`memoryview`-based variant of

View File

@ -772,7 +772,7 @@ Optimizations
* :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
:func:`shutil.copytree` and :func:`shutil.move` use platform-specific
"fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file
"fast-copy" syscalls on Linux and macOS in order to copy the file
more efficiently.
"fast-copy" means that the copying operation occurs within the kernel,
avoiding the use of userspace buffers in Python as in

View File

@ -50,7 +50,7 @@ elif _WINDOWS:
import nt
COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
_HAS_SENDFILE = posix and hasattr(os, "sendfile")
_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
_HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS
__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
@ -111,7 +111,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags):
def _fastcopy_sendfile(fsrc, fdst):
"""Copy data from one regular mmap-like fd to another by using
high-performance sendfile(2) syscall.
This should work on Linux >= 2.6.33 and Solaris only.
This should work on Linux >= 2.6.33 only.
"""
# Note: copyfileobj() is left alone in order to not introduce any
# unexpected breakage. Possible risks by using zero-copy calls
@ -122,7 +122,7 @@ def _fastcopy_sendfile(fsrc, fdst):
# GzipFile (which decompresses data), HTTPResponse (which decodes
# chunks).
# - possibly others (e.g. encrypted fs/partition?)
global _HAS_SENDFILE
global _USE_CP_SENDFILE
try:
infd = fsrc.fileno()
outfd = fdst.fileno()
@ -152,7 +152,7 @@ def _fastcopy_sendfile(fsrc, fdst):
# sendfile() on this platform (probably Linux < 2.6.33)
# does not support copies between regular files (only
# sockets).
_HAS_SENDFILE = False
_USE_CP_SENDFILE = False
raise _GiveupOnFastCopy(err)
if err.errno == errno.ENOSPC: # filesystem is full
@ -260,8 +260,8 @@ def copyfile(src, dst, *, follow_symlinks=True):
return dst
except _GiveupOnFastCopy:
pass
# Linux / Solaris
elif _HAS_SENDFILE:
# Linux
elif _USE_CP_SENDFILE:
try:
_fastcopy_sendfile(fsrc, fdst)
return dst

View File

@ -2315,7 +2315,7 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
# Emulate a case where sendfile() only support file->socket
# fds. In such a case copyfile() is supposed to skip the
# fast-copy attempt from then on.
assert shutil._HAS_SENDFILE
assert shutil._USE_CP_SENDFILE
try:
with unittest.mock.patch(
self.PATCHPOINT,
@ -2324,13 +2324,13 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
with self.assertRaises(_GiveupOnFastCopy):
shutil._fastcopy_sendfile(src, dst)
assert m.called
assert not shutil._HAS_SENDFILE
assert not shutil._USE_CP_SENDFILE
with unittest.mock.patch(self.PATCHPOINT) as m:
shutil.copyfile(TESTFN, TESTFN2)
assert not m.called
finally:
shutil._HAS_SENDFILE = True
shutil._USE_CP_SENDFILE = True
@unittest.skipIf(not MACOS, 'macOS only')

View File

@ -4450,7 +4450,7 @@ data_received() being called before connection_made().
:func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
:func:`shutil.copytree` and :func:`shutil.move` use platform-specific
fast-copy syscalls on Linux, Solaris and macOS in order to copy the file
fast-copy syscalls on Linux and macOS in order to copy the file
more efficiently. On Windows :func:`shutil.copyfile` uses a bigger default
buffer size (1 MiB instead of 16 KiB) and a :func:`memoryview`-based variant
of :func:`shutil.copyfileobj` is used. The speedup for copying a 512MiB file