bpo-36610: shutil.copyfile(): use sendfile() on Linux only (GH-13675)
...and avoid using it on Solaris as it can raise EINVAL if offset is equal or bigger than the size of the file
This commit is contained in:
parent
a16387ab2d
commit
413d955f8e
|
@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``".
|
|||
|
||||
On macOS `fcopyfile`_ is used to copy the file content (not metadata).
|
||||
|
||||
On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports
|
||||
copies between 2 regular file descriptors :func:`os.sendfile` is used.
|
||||
On Linux :func:`os.sendfile` is used.
|
||||
|
||||
On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB
|
||||
instead of 64 KiB) and a :func:`memoryview`-based variant of
|
||||
|
|
|
@ -772,7 +772,7 @@ Optimizations
|
|||
|
||||
* :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
|
||||
:func:`shutil.copytree` and :func:`shutil.move` use platform-specific
|
||||
"fast-copy" syscalls on Linux, macOS and Solaris in order to copy the file
|
||||
"fast-copy" syscalls on Linux and macOS in order to copy the file
|
||||
more efficiently.
|
||||
"fast-copy" means that the copying operation occurs within the kernel,
|
||||
avoiding the use of userspace buffers in Python as in
|
||||
|
|
|
@ -50,7 +50,7 @@ elif _WINDOWS:
|
|||
import nt
|
||||
|
||||
COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024
|
||||
_HAS_SENDFILE = posix and hasattr(os, "sendfile")
|
||||
_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux")
|
||||
_HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS
|
||||
|
||||
__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
|
||||
|
@ -111,7 +111,7 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags):
|
|||
def _fastcopy_sendfile(fsrc, fdst):
|
||||
"""Copy data from one regular mmap-like fd to another by using
|
||||
high-performance sendfile(2) syscall.
|
||||
This should work on Linux >= 2.6.33 and Solaris only.
|
||||
This should work on Linux >= 2.6.33 only.
|
||||
"""
|
||||
# Note: copyfileobj() is left alone in order to not introduce any
|
||||
# unexpected breakage. Possible risks by using zero-copy calls
|
||||
|
@ -122,7 +122,7 @@ def _fastcopy_sendfile(fsrc, fdst):
|
|||
# GzipFile (which decompresses data), HTTPResponse (which decodes
|
||||
# chunks).
|
||||
# - possibly others (e.g. encrypted fs/partition?)
|
||||
global _HAS_SENDFILE
|
||||
global _USE_CP_SENDFILE
|
||||
try:
|
||||
infd = fsrc.fileno()
|
||||
outfd = fdst.fileno()
|
||||
|
@ -152,7 +152,7 @@ def _fastcopy_sendfile(fsrc, fdst):
|
|||
# sendfile() on this platform (probably Linux < 2.6.33)
|
||||
# does not support copies between regular files (only
|
||||
# sockets).
|
||||
_HAS_SENDFILE = False
|
||||
_USE_CP_SENDFILE = False
|
||||
raise _GiveupOnFastCopy(err)
|
||||
|
||||
if err.errno == errno.ENOSPC: # filesystem is full
|
||||
|
@ -260,8 +260,8 @@ def copyfile(src, dst, *, follow_symlinks=True):
|
|||
return dst
|
||||
except _GiveupOnFastCopy:
|
||||
pass
|
||||
# Linux / Solaris
|
||||
elif _HAS_SENDFILE:
|
||||
# Linux
|
||||
elif _USE_CP_SENDFILE:
|
||||
try:
|
||||
_fastcopy_sendfile(fsrc, fdst)
|
||||
return dst
|
||||
|
|
|
@ -2315,7 +2315,7 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
|
|||
# Emulate a case where sendfile() only support file->socket
|
||||
# fds. In such a case copyfile() is supposed to skip the
|
||||
# fast-copy attempt from then on.
|
||||
assert shutil._HAS_SENDFILE
|
||||
assert shutil._USE_CP_SENDFILE
|
||||
try:
|
||||
with unittest.mock.patch(
|
||||
self.PATCHPOINT,
|
||||
|
@ -2324,13 +2324,13 @@ class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase):
|
|||
with self.assertRaises(_GiveupOnFastCopy):
|
||||
shutil._fastcopy_sendfile(src, dst)
|
||||
assert m.called
|
||||
assert not shutil._HAS_SENDFILE
|
||||
assert not shutil._USE_CP_SENDFILE
|
||||
|
||||
with unittest.mock.patch(self.PATCHPOINT) as m:
|
||||
shutil.copyfile(TESTFN, TESTFN2)
|
||||
assert not m.called
|
||||
finally:
|
||||
shutil._HAS_SENDFILE = True
|
||||
shutil._USE_CP_SENDFILE = True
|
||||
|
||||
|
||||
@unittest.skipIf(not MACOS, 'macOS only')
|
||||
|
|
|
@ -4450,7 +4450,7 @@ data_received() being called before connection_made().
|
|||
|
||||
:func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`,
|
||||
:func:`shutil.copytree` and :func:`shutil.move` use platform-specific
|
||||
fast-copy syscalls on Linux, Solaris and macOS in order to copy the file
|
||||
fast-copy syscalls on Linux and macOS in order to copy the file
|
||||
more efficiently. On Windows :func:`shutil.copyfile` uses a bigger default
|
||||
buffer size (1 MiB instead of 16 KiB) and a :func:`memoryview`-based variant
|
||||
of :func:`shutil.copyfileobj` is used. The speedup for copying a 512MiB file
|
||||
|
|
Loading…
Reference in New Issue