traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #6215: backport the 3.1 io lib

This commit is contained in:
Antoine Pitrou 2009-06-12 20:14:08 +00:00
parent 55bd1efb2a
commit 1969059327
21 changed files with 11865 additions and 3018 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Doc/library/io.rst
View File

@ -573,6 +573,10 @@ Text I/O
The name of the encoding used to decode the stream's bytes into
strings, and to encode strings into bytes.
.. attribute:: errors
The error setting of the decoder or encoder.
.. attribute:: newlines
A string, a tuple of strings, or ``None``, indicating the newlines
@ -625,13 +629,9 @@ Text I/O
If *line_buffering* is ``True``, :meth:`flush` is implied when a call to
write contains a newline character.
:class:`TextIOWrapper` provides these data attributes in addition to those of
:class:`TextIOWrapper` provides one attribute in addition to those of
:class:`TextIOBase` and its parents:
.. attribute:: errors
The encoding and decoding error setting.
.. attribute:: line_buffering
Whether line buffering is enabled.

1962
Lib/_pyio.py Normal file
View File

File diff suppressed because it is too large Load Diff

1835
Lib/io.py
View File

File diff suppressed because it is too large Load Diff

43
Lib/test/test_bufio.py
View File

@ -1,12 +1,15 @@
import unittest
from test import test_support
from test import test_support as support
# Simple test to ensure that optimizations in fileobject.c deliver
# the expected results. For best testing, run this under a debug-build
# Python too (to exercise asserts in the C code).
import io # C implementation.
import _pyio as pyio # Python implementation.
lengths = range(1, 257) + [512, 1000, 1024, 2048, 4096, 8192, 10000,
16384, 32768, 65536, 1000000]
# Simple test to ensure that optimizations in the IO library deliver the
# expected results. For best testing, run this under a debug-build Python too
# (to exercise asserts in the C code).
lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000,
16384, 32768, 65536, 1000000]
class BufferSizeTest(unittest.TestCase):
def try_one(self, s):
@ -14,27 +17,27 @@ class BufferSizeTest(unittest.TestCase):
# .readline()s deliver what we wrote.
# Ensure we can open TESTFN for writing.
test_support.unlink(test_support.TESTFN)
support.unlink(support.TESTFN)
# Since C doesn't guarantee we can write/read arbitrary bytes in text
# files, use binary mode.
f = open(test_support.TESTFN, "wb")
f = self.open(support.TESTFN, "wb")
try:
# write once with \n and once without
f.write(s)
f.write("\n")
f.write(b"\n")
f.write(s)
f.close()
f = open(test_support.TESTFN, "rb")
f = open(support.TESTFN, "rb")
line = f.readline()
self.assertEqual(line, s + "\n")
self.assertEqual(line, s + b"\n")
line = f.readline()
self.assertEqual(line, s)
line = f.readline()
self.assert_(not line) # Must be at EOF
f.close()
finally:
test_support.unlink(test_support.TESTFN)
support.unlink(support.TESTFN)
def drive_one(self, pattern):
for length in lengths:
@ -47,19 +50,27 @@ class BufferSizeTest(unittest.TestCase):
teststring = pattern * q + pattern[:r]
self.assertEqual(len(teststring), length)
self.try_one(teststring)
self.try_one(teststring + "x")
self.try_one(teststring + b"x")
self.try_one(teststring[:-1])
def test_primepat(self):
# A pattern with prime length, to avoid simple relationships with
# stdio buffer sizes.
self.drive_one("1234567890\00\01\02\03\04\05\06")
self.drive_one(b"1234567890\00\01\02\03\04\05\06")
def test_nullpat(self):
self.drive_one("\0" * 1000)
self.drive_one(bytes(1000))
class CBufferSizeTest(BufferSizeTest):
open = io.open
class PyBufferSizeTest(BufferSizeTest):
open = staticmethod(pyio.open)
def test_main():
test_support.run_unittest(BufferSizeTest)
support.run_unittest(CBufferSizeTest, PyBufferSizeTest)
if __name__ == "__main__":
test_main()

423
Lib/test/test_file.py
View File

@ -1,13 +1,14 @@
from __future__ import print_function
import sys
import os
import unittest
import itertools
import time
import threading
from array import array
from weakref import proxy
from test import test_support
import io
import _pyio as pyio
from test.test_support import TESTFN, findfile, run_unittest
from UserList import UserList
@ -15,7 +16,7 @@ class AutoFileTests(unittest.TestCase):
# file tests for which a test file is automatically set up
def setUp(self):
self.f = open(TESTFN, 'wb')
self.f = self.open(TESTFN, 'wb')
def tearDown(self):
if self.f:
@ -25,7 +26,7 @@ class AutoFileTests(unittest.TestCase):
def testWeakRefs(self):
# verify weak references
p = proxy(self.f)
p.write('teststring')
p.write(b'teststring')
self.assertEquals(self.f.tell(), p.tell())
self.f.close()
self.f = None
@ -34,35 +35,35 @@ class AutoFileTests(unittest.TestCase):
def testAttributes(self):
# verify expected attributes exist
f = self.f
softspace = f.softspace
f.name # merely shouldn't blow up
f.mode # ditto
f.closed # ditto
# verify softspace is writable
f.softspace = softspace # merely shouldn't blow up
# verify the others aren't
for attr in 'name', 'mode', 'closed':
self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops')
def testReadinto(self):
# verify readinto
self.f.write('12')
self.f.write(b'12')
self.f.close()
a = array('c', 'x'*10)
self.f = open(TESTFN, 'rb')
a = array('b', b'x'*10)
self.f = self.open(TESTFN, 'rb')
n = self.f.readinto(a)
self.assertEquals('12', a.tostring()[:n])
self.assertEquals(b'12', a.tostring()[:n])
def testReadinto_text(self):
# verify readinto refuses text files
a = array('b', b'x'*10)
self.f.close()
self.f = self.open(TESTFN, 'r')
if hasattr(self.f, "readinto"):
self.assertRaises(TypeError, self.f.readinto, a)
def testWritelinesUserList(self):
# verify writelines with instance sequence
l = UserList(['1', '2'])
l = UserList([b'1', b'2'])
self.f.writelines(l)
self.f.close()
self.f = open(TESTFN, 'rb')
self.f = self.open(TESTFN, 'rb')
buf = self.f.read()
self.assertEquals(buf, '12')
self.assertEquals(buf, b'12')
def testWritelinesIntegers(self):
# verify writelines with integers
@ -81,36 +82,43 @@ class AutoFileTests(unittest.TestCase):
self.assertRaises(TypeError, self.f.writelines,
[NonString(), NonString()])
def testRepr(self):
# verify repr works
self.assert_(repr(self.f).startswith("<open file '" + TESTFN))
def testErrors(self):
f = self.f
self.assertEquals(f.name, TESTFN)
self.assert_(not f.isatty())
self.assert_(not f.closed)
self.assertRaises(TypeError, f.readinto, "")
if hasattr(f, "readinto"):
self.assertRaises((IOError, TypeError), f.readinto, "")
f.close()
self.assert_(f.closed)
def testMethods(self):
methods = ['fileno', 'flush', 'isatty', 'next', 'read', 'readinto',
'readline', 'readlines', 'seek', 'tell', 'truncate',
'write', 'xreadlines', '__iter__']
if sys.platform.startswith('atheos'):
methods.remove('truncate')
methods = [('fileno', ()),
('flush', ()),
('isatty', ()),
('next', ()),
('read', ()),
('write', (b"",)),
('readline', ()),
('readlines', ()),
('seek', (0,)),
('tell', ()),
('write', (b"",)),
('writelines', ([],)),
('__iter__', ()),
]
if not sys.platform.startswith('atheos'):
methods.append(('truncate', ()))
# __exit__ should close the file
self.f.__exit__(None, None, None)
self.assert_(self.f.closed)
for methodname in methods:
for methodname, args in methods:
method = getattr(self.f, methodname)
# should raise on closed file
self.assertRaises(ValueError, method)
self.assertRaises(ValueError, self.f.writelines, [])
self.assertRaises(ValueError, method, *args)
# file is closed, __exit__ shouldn't do anything
self.assertEquals(self.f.__exit__(None, None, None), None)
@ -123,70 +131,47 @@ class AutoFileTests(unittest.TestCase):
def testReadWhenWriting(self):
self.assertRaises(IOError, self.f.read)
class OtherFileTests(unittest.TestCase):
class CAutoFileTests(AutoFileTests):
open = io.open
def testOpenDir(self):
this_dir = os.path.dirname(__file__)
for mode in (None, "w"):
try:
if mode:
f = open(this_dir, mode)
else:
f = open(this_dir)
except IOError as e:
self.assertEqual(e.filename, this_dir)
else:
self.fail("opening a directory didn't raise an IOError")
class PyAutoFileTests(AutoFileTests):
open = staticmethod(pyio.open)
class OtherFileTests(unittest.TestCase):
def testModeStrings(self):
# check invalid mode strings
for mode in ("", "aU", "wU+"):
try:
f = open(TESTFN, mode)
f = self.open(TESTFN, mode)
except ValueError:
pass
else:
f.close()
self.fail('%r is an invalid file mode' % mode)
# Some invalid modes fail on Windows, but pass on Unix
# Issue3965: avoid a crash on Windows when filename is unicode
for name in (TESTFN, unicode(TESTFN), unicode(TESTFN + '\t')):
try:
f = open(name, "rr")
except (IOError, ValueError):
pass
else:
f.close()
def testStdin(self):
# This causes the interpreter to exit on OSF1 v5.1.
if sys.platform != 'osf1V5':
self.assertRaises(IOError, sys.stdin.seek, -1)
self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
else:
print >>sys.__stdout__, (
print((
' Skipping sys.stdin.seek(-1), it may crash the interpreter.'
' Test manually.')
self.assertRaises(IOError, sys.stdin.truncate)
def testUnicodeOpen(self):
# verify repr works for unicode too
f = open(unicode(TESTFN), "w")
self.assert_(repr(f).startswith("<open file u'" + TESTFN))
f.close()
os.unlink(TESTFN)
' Test manually.'), file=sys.__stdout__)
self.assertRaises((IOError, ValueError), sys.stdin.truncate)
def testBadModeArgument(self):
# verify that we get a sensible error message for bad mode argument
bad_mode = "qwerty"
try:
f = open(TESTFN, bad_mode)
except ValueError, msg:
if msg[0] != 0:
f = self.open(TESTFN, bad_mode)
except ValueError as msg:
if msg.args[0] != 0:
s = str(msg)
if s.find(TESTFN) != -1 or s.find(bad_mode) == -1:
self.fail("bad error message for invalid mode: %s" % s)
# if msg[0] == 0, we're probably on Windows where there may be
# if msg.args[0] == 0, we're probably on Windows where there may be
# no obvious way to discover why open() failed.
else:
f.close()
@ -197,31 +182,32 @@ class OtherFileTests(unittest.TestCase):
# misbehaviour especially with repeated close() calls
for s in (-1, 0, 1, 512):
try:
f = open(TESTFN, 'w', s)
f.write(str(s))
f = self.open(TESTFN, 'wb', s)
f.write(str(s).encode("ascii"))
f.close()
f.close()
f = open(TESTFN, 'r', s)
d = int(f.read())
f = self.open(TESTFN, 'rb', s)
d = int(f.read().decode("ascii"))
f.close()
f.close()
except IOError, msg:
except IOError as msg:
self.fail('error setting buffer size %d: %s' % (s, str(msg)))
self.assertEquals(d, s)
def testTruncateOnWindows(self):
os.unlink(TESTFN)
# SF bug <http://www.python.org/sf/801631>
# "file.truncate fault on windows"
def bug801631():
# SF bug <http://www.python.org/sf/801631>
# "file.truncate fault on windows"
f = open(TESTFN, 'wb')
f.write('12345678901') # 11 bytes
os.unlink(TESTFN)
f = self.open(TESTFN, 'wb')
try:
f.write(b'12345678901') # 11 bytes
f.close()
f = open(TESTFN,'rb+')
f = self.open(TESTFN,'rb+')
data = f.read(5)
if data != '12345':
if data != b'12345':
self.fail("Read on file opened for update failed %r" % data)
if f.tell() != 5:
self.fail("File pos after read wrong %d" % f.tell())
@ -234,56 +220,42 @@ class OtherFileTests(unittest.TestCase):
size = os.path.getsize(TESTFN)
if size != 5:
self.fail("File size after ftruncate wrong %d" % size)
try:
bug801631()
finally:
f.close()
os.unlink(TESTFN)
def testIteration(self):
# Test the complex interaction when mixing file-iteration and the
# various read* methods. Ostensibly, the mixture could just be tested
# to work when it should work according to the Python language,
# instead of fail when it should fail according to the current CPython
# implementation. People don't always program Python the way they
# should, though, and the implemenation might change in subtle ways,
# so we explicitly test for errors, too; the test will just have to
# be updated when the implementation changes.
# various read* methods.
dataoffset = 16384
filler = "ham\n"
filler = b"ham\n"
assert not dataoffset % len(filler), \
"dataoffset must be multiple of len(filler)"
nchunks = dataoffset // len(filler)
testlines = [
"spam, spam and eggs\n",
"eggs, spam, ham and spam\n",
"saussages, spam, spam and eggs\n",
"spam, ham, spam and eggs\n",
"spam, spam, spam, spam, spam, ham, spam\n",
"wonderful spaaaaaam.\n"
b"spam, spam and eggs\n",
b"eggs, spam, ham and spam\n",
b"saussages, spam, spam and eggs\n",
b"spam, ham, spam and eggs\n",
b"spam, spam, spam, spam, spam, ham, spam\n",
b"wonderful spaaaaaam.\n"
]
methods = [("readline", ()), ("read", ()), ("readlines", ()),
("readinto", (array("c", " "*100),))]
("readinto", (array("b", b" "*100),))]
try:
# Prepare the testfile
bag = open(TESTFN, "w")
bag = self.open(TESTFN, "wb")
bag.write(filler * nchunks)
bag.writelines(testlines)
bag.close()
# Test for appropriate errors mixing read* and iteration
for methodname, args in methods:
f = open(TESTFN)
if f.next() != filler:
f = self.open(TESTFN, 'rb')
if next(f) != filler:
self.fail, "Broken testfile"
meth = getattr(f, methodname)
try:
meth(*args)
except ValueError:
pass
else:
self.fail("%s%r after next() didn't raise ValueError" %
(methodname, args))
meth(*args) # This simply shouldn't fail
f.close()
# Test to see if harmless (by accident) mixing of read* and
@ -293,9 +265,9 @@ class OtherFileTests(unittest.TestCase):
# ("h", "a", "m", "\n"), so 4096 lines of that should get us
# exactly on the buffer boundary for any power-of-2 buffersize
# between 4 and 16384 (inclusive).
f = open(TESTFN)
f = self.open(TESTFN, 'rb')
for i in range(nchunks):
f.next()
next(f)
testline = testlines.pop(0)
try:
line = f.readline()
@ -306,7 +278,7 @@ class OtherFileTests(unittest.TestCase):
self.fail("readline() after next() with empty buffer "
"failed. Got %r, expected %r" % (line, testline))
testline = testlines.pop(0)
buf = array("c", "\x00" * len(testline))
buf = array("b", b"\x00" * len(testline))
try:
f.readinto(buf)
except ValueError:
@ -335,7 +307,7 @@ class OtherFileTests(unittest.TestCase):
self.fail("readlines() after next() with empty buffer "
"failed. Got %r, expected %r" % (line, testline))
# Reading after iteration hit EOF shouldn't hurt either
f = open(TESTFN)
f = self.open(TESTFN, 'rb')
try:
for line in f:
pass
@ -351,222 +323,19 @@ class OtherFileTests(unittest.TestCase):
finally:
os.unlink(TESTFN)
class FileSubclassTests(unittest.TestCase):
class COtherFileTests(OtherFileTests):
open = io.open
def testExit(self):
# test that exiting with context calls subclass' close
class C(file):
def __init__(self, *args):
self.subclass_closed = False
file.__init__(self, *args)
def close(self):
self.subclass_closed = True
file.close(self)
with C(TESTFN, 'w') as f:
pass
self.failUnless(f.subclass_closed)
class FileThreadingTests(unittest.TestCase):
# These tests check the ability to call various methods of file objects
# (including close()) concurrently without crashing the Python interpreter.
# See #815646, #595601
def setUp(self):
self.f = None
self.filename = TESTFN
with open(self.filename, "w") as f:
f.write("\n".join("0123456789"))
self._count_lock = threading.Lock()
self.close_count = 0
self.close_success_count = 0
def tearDown(self):
if self.f:
try:
self.f.close()
except (EnvironmentError, ValueError):
pass
try:
os.remove(self.filename)
except EnvironmentError:
pass
def _create_file(self):
self.f = open(self.filename, "w+")
def _close_file(self):
with self._count_lock:
self.close_count += 1
self.f.close()
with self._count_lock:
self.close_success_count += 1
def _close_and_reopen_file(self):
self._close_file()
# if close raises an exception thats fine, self.f remains valid so
# we don't need to reopen.
self._create_file()
def _run_workers(self, func, nb_workers, duration=0.2):
with self._count_lock:
self.close_count = 0
self.close_success_count = 0
self.do_continue = True
threads = []
try:
for i in range(nb_workers):
t = threading.Thread(target=func)
t.start()
threads.append(t)
for _ in xrange(100):
time.sleep(duration/100)
with self._count_lock:
if self.close_count-self.close_success_count > nb_workers+1:
if test_support.verbose:
print 'Q',
break
time.sleep(duration)
finally:
self.do_continue = False
for t in threads:
t.join()
def _test_close_open_io(self, io_func, nb_workers=5):
def worker():
self._create_file()
funcs = itertools.cycle((
lambda: io_func(),
lambda: self._close_and_reopen_file(),
))
for f in funcs:
if not self.do_continue:
break
try:
f()
except (IOError, ValueError):
pass
self._run_workers(worker, nb_workers)
if test_support.verbose:
# Useful verbose statistics when tuning this test to take
# less time to run but still ensuring that its still useful.
#
# the percent of close calls that raised an error
percent = 100. - 100.*self.close_success_count/self.close_count
print self.close_count, ('%.4f ' % percent),
def test_close_open(self):
def io_func():
pass
self._test_close_open_io(io_func)
def test_close_open_flush(self):
def io_func():
self.f.flush()
self._test_close_open_io(io_func)
def test_close_open_iter(self):
def io_func():
list(iter(self.f))
self._test_close_open_io(io_func)
def test_close_open_isatty(self):
def io_func():
self.f.isatty()
self._test_close_open_io(io_func)
def test_close_open_print(self):
def io_func():
print >> self.f, ''
self._test_close_open_io(io_func)
def test_close_open_read(self):
def io_func():
self.f.read(0)
self._test_close_open_io(io_func)
def test_close_open_readinto(self):
def io_func():
a = array('c', 'xxxxx')
self.f.readinto(a)
self._test_close_open_io(io_func)
def test_close_open_readline(self):
def io_func():
self.f.readline()
self._test_close_open_io(io_func)
def test_close_open_readlines(self):
def io_func():
self.f.readlines()
self._test_close_open_io(io_func)
def test_close_open_seek(self):
def io_func():
self.f.seek(0, 0)
self._test_close_open_io(io_func)
def test_close_open_tell(self):
def io_func():
self.f.tell()
self._test_close_open_io(io_func)
def test_close_open_truncate(self):
def io_func():
self.f.truncate()
self._test_close_open_io(io_func)
def test_close_open_write(self):
def io_func():
self.f.write('')
self._test_close_open_io(io_func)
def test_close_open_writelines(self):
def io_func():
self.f.writelines('')
self._test_close_open_io(io_func)
class StdoutTests(unittest.TestCase):
def test_move_stdout_on_write(self):
# Issue 3242: sys.stdout can be replaced (and freed) during a
# print statement; prevent a segfault in this case
save_stdout = sys.stdout
class File:
def write(self, data):
if '\n' in data:
sys.stdout = save_stdout
try:
sys.stdout = File()
print "some text"
finally:
sys.stdout = save_stdout
def test_del_stdout_before_print(self):
# Issue 4597: 'print' with no argument wasn't reporting when
# sys.stdout was deleted.
save_stdout = sys.stdout
del sys.stdout
try:
print
except RuntimeError as e:
self.assertEquals(str(e), "lost sys.stdout")
else:
self.fail("Expected RuntimeError")
finally:
sys.stdout = save_stdout
class PyOtherFileTests(OtherFileTests):
open = staticmethod(pyio.open)
def test_main():
# Historically, these tests have been sloppy about removing TESTFN.
# So get rid of it no matter what.
try:
run_unittest(AutoFileTests, OtherFileTests, FileSubclassTests,
FileThreadingTests, StdoutTests)
run_unittest(CAutoFileTests, PyAutoFileTests,
COtherFileTests, PyOtherFileTests)
finally:
if os.path.exists(TESTFN):
os.unlink(TESTFN)

185
Lib/test/test_fileio.py
View File

@ -1,23 +1,26 @@
# Adapted from test_file.py by Daniel Stutzbach
#from __future__ import unicode_literals
from __future__ import unicode_literals
import sys
import os
import errno
import unittest
from array import array
from weakref import proxy
from functools import wraps
from test.test_support import (TESTFN, findfile, check_warnings, run_unittest,
make_bad_fd)
from UserList import UserList
from test.test_support import py3k_bytes as bytes
import _fileio
from _io import FileIO as _FileIO
class AutoFileTests(unittest.TestCase):
# file tests for which a test file is automatically set up
def setUp(self):
self.f = _fileio._FileIO(TESTFN, 'w')
self.f = _FileIO(TESTFN, 'w')
def tearDown(self):
if self.f:
@ -34,7 +37,7 @@ class AutoFileTests(unittest.TestCase):
self.assertRaises(ReferenceError, getattr, p, 'tell')
def testSeekTell(self):
self.f.write(bytes(bytearray(range(20))))
self.f.write(bytes(range(20)))
self.assertEquals(self.f.tell(), 20)
self.f.seek(0)
self.assertEquals(self.f.tell(), 0)
@ -61,17 +64,21 @@ class AutoFileTests(unittest.TestCase):
def testReadinto(self):
# verify readinto
self.f.write(bytes(bytearray([1, 2])))
self.f.write(b"\x01\x02")
self.f.close()
a = array('b', b'x'*10)
self.f = _fileio._FileIO(TESTFN, 'r')
a = array(b'b', b'x'*10)
self.f = _FileIO(TESTFN, 'r')
n = self.f.readinto(a)
self.assertEquals(array('b', [1, 2]), a[:n])
self.assertEquals(array(b'b', [1, 2]), a[:n])
def testRepr(self):
self.assertEquals(repr(self.f),
"_fileio._FileIO(%d, %s)" % (self.f.fileno(),
repr(self.f.mode)))
self.assertEquals(repr(self.f), "<_io.FileIO name=%r mode='%s'>"
% (self.f.name, self.f.mode))
del self.f.name
self.assertEquals(repr(self.f), "<_io.FileIO fd=%r mode='%s'>"
% (self.f.fileno(), self.f.mode))
self.f.close()
self.assertEquals(repr(self.f), "<_io.FileIO [closed]>")
def testErrors(self):
f = self.f
@ -81,7 +88,7 @@ class AutoFileTests(unittest.TestCase):
self.assertRaises(ValueError, f.read, 10) # Open for reading
f.close()
self.assert_(f.closed)
f = _fileio._FileIO(TESTFN, 'r')
f = _FileIO(TESTFN, 'r')
self.assertRaises(TypeError, f.readinto, "")
self.assert_(not f.closed)
f.close()
@ -107,31 +114,131 @@ class AutoFileTests(unittest.TestCase):
# Windows always returns "[Errno 13]: Permission denied
# Unix calls dircheck() and returns "[Errno 21]: Is a directory"
try:
_fileio._FileIO('.', 'r')
_FileIO('.', 'r')
except IOError as e:
self.assertNotEqual(e.errno, 0)
self.assertEqual(e.filename, ".")
else:
self.fail("Should have raised IOError")
#A set of functions testing that we get expected behaviour if someone has
#manually closed the internal file descriptor. First, a decorator:
def ClosedFD(func):
@wraps(func)
def wrapper(self):
#forcibly close the fd before invoking the problem function
f = self.f
os.close(f.fileno())
try:
func(self, f)
finally:
try:
self.f.close()
except IOError:
pass
return wrapper
def ClosedFDRaises(func):
@wraps(func)
def wrapper(self):
#forcibly close the fd before invoking the problem function
f = self.f
os.close(f.fileno())
try:
func(self, f)
except IOError as e:
self.assertEqual(e.errno, errno.EBADF)
else:
self.fail("Should have raised IOError")
finally:
try:
self.f.close()
except IOError:
pass
return wrapper
@ClosedFDRaises
def testErrnoOnClose(self, f):
f.close()
@ClosedFDRaises
def testErrnoOnClosedWrite(self, f):
f.write('a')
@ClosedFDRaises
def testErrnoOnClosedSeek(self, f):
f.seek(0)
@ClosedFDRaises
def testErrnoOnClosedTell(self, f):
f.tell()
@ClosedFDRaises
def testErrnoOnClosedTruncate(self, f):
f.truncate(0)
@ClosedFD
def testErrnoOnClosedSeekable(self, f):
f.seekable()
@ClosedFD
def testErrnoOnClosedReadable(self, f):
f.readable()
@ClosedFD
def testErrnoOnClosedWritable(self, f):
f.writable()
@ClosedFD
def testErrnoOnClosedFileno(self, f):
f.fileno()
@ClosedFD
def testErrnoOnClosedIsatty(self, f):
self.assertEqual(f.isatty(), False)
def ReopenForRead(self):
try:
self.f.close()
except IOError:
pass
self.f = _FileIO(TESTFN, 'r')
os.close(self.f.fileno())
return self.f
@ClosedFDRaises
def testErrnoOnClosedRead(self, f):
f = self.ReopenForRead()
f.read(1)
@ClosedFDRaises
def testErrnoOnClosedReadall(self, f):
f = self.ReopenForRead()
f.readall()
@ClosedFDRaises
def testErrnoOnClosedReadinto(self, f):
f = self.ReopenForRead()
a = array(b'b', b'x'*10)
f.readinto(a)
class OtherFileTests(unittest.TestCase):
def testAbles(self):
try:
f = _fileio._FileIO(TESTFN, "w")
f = _FileIO(TESTFN, "w")
self.assertEquals(f.readable(), False)
self.assertEquals(f.writable(), True)
self.assertEquals(f.seekable(), True)
f.close()
f = _fileio._FileIO(TESTFN, "r")
f = _FileIO(TESTFN, "r")
self.assertEquals(f.readable(), True)
self.assertEquals(f.writable(), False)
self.assertEquals(f.seekable(), True)
f.close()
f = _fileio._FileIO(TESTFN, "a+")
f = _FileIO(TESTFN, "a+")
self.assertEquals(f.readable(), True)
self.assertEquals(f.writable(), True)
self.assertEquals(f.seekable(), True)
@ -140,14 +247,14 @@ class OtherFileTests(unittest.TestCase):
if sys.platform != "win32":
try:
f = _fileio._FileIO("/dev/tty", "a")
f = _FileIO("/dev/tty", "a")
except EnvironmentError:
# When run in a cron job there just aren't any
# ttys, so skip the test. This also handles other
# OS'es that don't support /dev/tty.
pass
else:
f = _fileio._FileIO("/dev/tty", "a")
f = _FileIO("/dev/tty", "a")
self.assertEquals(f.readable(), False)
self.assertEquals(f.writable(), True)
if sys.platform != "darwin" and \
@ -164,7 +271,7 @@ class OtherFileTests(unittest.TestCase):
# check invalid mode strings
for mode in ("", "aU", "wU+", "rw", "rt"):
try:
f = _fileio._FileIO(TESTFN, mode)
f = _FileIO(TESTFN, mode)
except ValueError:
pass
else:
@ -173,19 +280,35 @@ class OtherFileTests(unittest.TestCase):
def testUnicodeOpen(self):
# verify repr works for unicode too
f = _fileio._FileIO(str(TESTFN), "w")
f = _FileIO(str(TESTFN), "w")
f.close()
os.unlink(TESTFN)
def testBytesOpen(self):
# Opening a bytes filename
try:
fn = TESTFN.encode("ascii")
except UnicodeEncodeError:
# Skip test
return
f = _FileIO(fn, "w")
try:
f.write(b"abc")
f.close()
with open(TESTFN, "rb") as f:
self.assertEquals(f.read(), b"abc")
finally:
os.unlink(TESTFN)
def testInvalidFd(self):
self.assertRaises(ValueError, _fileio._FileIO, -10)
self.assertRaises(OSError, _fileio._FileIO, make_bad_fd())
self.assertRaises(ValueError, _FileIO, -10)
self.assertRaises(OSError, _FileIO, make_bad_fd())
def testBadModeArgument(self):
# verify that we get a sensible error message for bad mode argument
bad_mode = "qwerty"
try:
f = _fileio._FileIO(TESTFN, bad_mode)
f = _FileIO(TESTFN, bad_mode)
except ValueError as msg:
if msg.args[0] != 0:
s = str(msg)
@ -201,13 +324,13 @@ class OtherFileTests(unittest.TestCase):
def bug801631():
# SF bug <http://www.python.org/sf/801631>
# "file.truncate fault on windows"
f = _fileio._FileIO(TESTFN, 'w')
f.write(bytes(bytearray(range(11))))
f = _FileIO(TESTFN, 'w')
f.write(bytes(range(11)))
f.close()
f = _fileio._FileIO(TESTFN,'r+')
f = _FileIO(TESTFN,'r+')
data = f.read(5)
if data != bytes(bytearray(range(5))):
if data != bytes(range(5)):
self.fail("Read on file opened for update failed %r" % data)
if f.tell() != 5:
self.fail("File pos after read wrong %d" % f.tell())
@ -245,14 +368,14 @@ class OtherFileTests(unittest.TestCase):
pass
def testInvalidInit(self):
self.assertRaises(TypeError, _fileio._FileIO, "1", 0, 0)
self.assertRaises(TypeError, _FileIO, "1", 0, 0)
def testWarnings(self):
with check_warnings() as w:
self.assertEqual(w.warnings, [])
self.assertRaises(TypeError, _fileio._FileIO, [])
self.assertRaises(TypeError, _FileIO, [])
self.assertEqual(w.warnings, [])
self.assertRaises(ValueError, _fileio._FileIO, "/some/invalid/name", "rt")
self.assertRaises(ValueError, _FileIO, "/some/invalid/name", "rt")
self.assertEqual(w.warnings, [])

1964
Lib/test/test_io.py
View File

File diff suppressed because it is too large Load Diff

89
Lib/test/test_largefile.py
View File

@ -1,12 +1,16 @@
"""Test largefile support on system where this makes sense.
"""
from __future__ import print_function
import os
import stat
import sys
import unittest
from test.test_support import run_unittest, TESTFN, verbose, requires, \
unlink
import io # C implementation of io
import _pyio as pyio # Python implementation of io
try:
import signal
@ -18,10 +22,10 @@ except (ImportError, AttributeError):
pass
# create >2GB file (2GB = 2147483648 bytes)
size = 2500000000L
size = 2500000000
class TestCase(unittest.TestCase):
class LargeFileTest(unittest.TestCase):
"""Test that each file function works as expected for a large
(i.e. > 2GB, do we have to check > 4GB) files.
@ -33,28 +37,28 @@ class TestCase(unittest.TestCase):
def test_seek(self):
if verbose:
print 'create large file via seek (may be sparse file) ...'
with open(TESTFN, 'wb') as f:
f.write('z')
print('create large file via seek (may be sparse file) ...')
with self.open(TESTFN, 'wb') as f:
f.write(b'z')
f.seek(0)
f.seek(size)
f.write('a')
f.write(b'a')
f.flush()
if verbose:
print 'check file size with os.fstat'
print('check file size with os.fstat')
self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1)
def test_osstat(self):
if verbose:
print 'check file size with os.stat'
print('check file size with os.stat')
self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
def test_seek_read(self):
if verbose:
print 'play around with seek() and read() with the built largefile'
with open(TESTFN, 'rb') as f:
print('play around with seek() and read() with the built largefile')
with self.open(TESTFN, 'rb') as f:
self.assertEqual(f.tell(), 0)
self.assertEqual(f.read(1), 'z')
self.assertEqual(f.read(1), b'z')
self.assertEqual(f.tell(), 1)
f.seek(0)
self.assertEqual(f.tell(), 0)
@ -77,15 +81,15 @@ class TestCase(unittest.TestCase):
f.seek(size)
self.assertEqual(f.tell(), size)
# the 'a' that was written at the end of file above
self.assertEqual(f.read(1), 'a')
self.assertEqual(f.read(1), b'a')
f.seek(-size-1, 1)
self.assertEqual(f.read(1), 'z')
self.assertEqual(f.read(1), b'z')
self.assertEqual(f.tell(), 1)
def test_lseek(self):
if verbose:
print 'play around with os.lseek() with the built largefile'
with open(TESTFN, 'rb') as f:
print('play around with os.lseek() with the built largefile')
with self.open(TESTFN, 'rb') as f:
self.assertEqual(os.lseek(f.fileno(), 0, 0), 0)
self.assertEqual(os.lseek(f.fileno(), 42, 0), 42)
self.assertEqual(os.lseek(f.fileno(), 42, 1), 84)
@ -95,16 +99,16 @@ class TestCase(unittest.TestCase):
self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0)
self.assertEqual(os.lseek(f.fileno(), size, 0), size)
# the 'a' that was written at the end of file above
self.assertEqual(f.read(1), 'a')
self.assertEqual(f.read(1), b'a')
def test_truncate(self):
if verbose:
print 'try truncate'
with open(TESTFN, 'r+b') as f:
print('try truncate')
with self.open(TESTFN, 'r+b') as f:
# this is already decided before start running the test suite
# but we do it anyway for extra protection
if not hasattr(f, 'truncate'):
raise unittest.SkipTest, "open().truncate() not available on this system"
raise unittest.SkipTest("open().truncate() not available on this system")
f.seek(0, 2)
# else we've lost track of the true size
self.assertEqual(f.tell(), size+1)
@ -120,17 +124,25 @@ class TestCase(unittest.TestCase):
newsize -= 1
f.seek(42)
f.truncate(newsize)
self.assertEqual(f.tell(), 42) # else pointer moved
f.seek(0, 2)
self.assertEqual(f.tell(), newsize) # else wasn't truncated
f.seek(0, 2)
self.assertEqual(f.tell(), newsize)
# XXX truncate(larger than true size) is ill-defined
# across platform; cut it waaaaay back
f.seek(0)
f.truncate(1)
self.assertEqual(f.tell(), 0) # else pointer moved
self.assertEqual(f.tell(), 1) # else pointer moved
f.seek(0)
self.assertEqual(len(f.read()), 1) # else wasn't truncated
def test_seekable(self):
# Issue #5016; seekable() can return False when the current position
# is negative when truncated to an int.
for pos in (2**31-1, 2**31, 2**31+1):
with self.open(TESTFN, 'rb') as f:
f.seek(pos)
self.assert_(f.seekable())
def test_main():
# On Windows and Mac OSX this test comsumes large resources; It
@ -144,34 +156,39 @@ def test_main():
# Only run if the current filesystem supports large files.
# (Skip this test on Windows, since we now always support
# large files.)
f = open(TESTFN, 'wb')
f = open(TESTFN, 'wb', buffering=0)
try:
# 2**31 == 2147483648
f.seek(2147483649L)
f.seek(2147483649)
# Seeking is not enough of a test: you must write and
# flush, too!
f.write("x")
f.write(b'x')
f.flush()
except (IOError, OverflowError):
f.close()
unlink(TESTFN)
raise unittest.SkipTest, "filesystem does not have largefile support"
raise unittest.SkipTest("filesystem does not have largefile support")
else:
f.close()
suite = unittest.TestSuite()
suite.addTest(TestCase('test_seek'))
suite.addTest(TestCase('test_osstat'))
suite.addTest(TestCase('test_seek_read'))
suite.addTest(TestCase('test_lseek'))
with open(TESTFN, 'w') as f:
if hasattr(f, 'truncate'):
suite.addTest(TestCase('test_truncate'))
unlink(TESTFN)
for _open, prefix in [(io.open, 'C'), (pyio.open, 'Py')]:
class TestCase(LargeFileTest):
pass
TestCase.open = staticmethod(_open)
TestCase.__name__ = prefix + LargeFileTest.__name__
suite.addTest(TestCase('test_seek'))
suite.addTest(TestCase('test_osstat'))
suite.addTest(TestCase('test_seek_read'))
suite.addTest(TestCase('test_lseek'))
with _open(TESTFN, 'wb') as f:
if hasattr(f, 'truncate'):
suite.addTest(TestCase('test_truncate'))
suite.addTest(TestCase('test_seekable'))
unlink(TESTFN)
try:
run_unittest(suite)
finally:
unlink(TESTFN)
if __name__ == '__main__':
test_main()

215
Lib/test/test_memoryio.py
View File

@ -4,23 +4,66 @@ BytesIO -- for bytes
"""
from __future__ import unicode_literals
from __future__ import print_function
import unittest
from test import test_support
from test import test_support as support
import io
import _pyio as pyio
import sys
import array
try:
import _bytesio
has_c_implementation = True
except ImportError:
has_c_implementation = False
class MemorySeekTestMixin:
def testInit(self):
buf = self.buftype("1234567890")
bytesIo = self.ioclass(buf)
def testRead(self):
buf = self.buftype("1234567890")
bytesIo = self.ioclass(buf)
self.assertEquals(buf[:1], bytesIo.read(1))
self.assertEquals(buf[1:5], bytesIo.read(4))
self.assertEquals(buf[5:], bytesIo.read(900))
self.assertEquals(self.EOF, bytesIo.read())
def testReadNoArgs(self):
buf = self.buftype("1234567890")
bytesIo = self.ioclass(buf)
self.assertEquals(buf, bytesIo.read())
self.assertEquals(self.EOF, bytesIo.read())
def testSeek(self):
buf = self.buftype("1234567890")
bytesIo = self.ioclass(buf)
bytesIo.read(5)
bytesIo.seek(0)
self.assertEquals(buf, bytesIo.read())
bytesIo.seek(3)
self.assertEquals(buf[3:], bytesIo.read())
self.assertRaises(TypeError, bytesIo.seek, 0.0)
def testTell(self):
buf = self.buftype("1234567890")
bytesIo = self.ioclass(buf)
self.assertEquals(0, bytesIo.tell())
bytesIo.seek(5)
self.assertEquals(5, bytesIo.tell())
bytesIo.seek(10000)
self.assertEquals(10000, bytesIo.tell())
class MemoryTestMixin:
def test_detach(self):
buf = self.ioclass()
self.assertRaises(self.UnsupportedOperation, buf.detach)
def write_ops(self, f, t):
self.assertEqual(f.write(t("blah.")), 5)
self.assertEqual(f.seek(0), 0)
@ -151,7 +194,7 @@ class MemoryTestMixin:
self.assertEqual(memio.readline(), self.EOF)
memio.seek(0)
self.assertEqual(type(memio.readline()), type(buf))
self.assertEqual(memio.readline(None), buf)
self.assertEqual(memio.readline(), buf)
self.assertRaises(TypeError, memio.readline, '')
memio.close()
self.assertRaises(ValueError, memio.readline)
@ -197,7 +240,7 @@ class MemoryTestMixin:
self.assertEqual(i, 10)
memio = self.ioclass(buf * 2)
memio.close()
self.assertRaises(ValueError, memio.next)
self.assertRaises(ValueError, next, memio)
def test_getvalue(self):
buf = self.buftype("1234567890")
@ -299,11 +342,14 @@ class MemoryTestMixin:
self.assertEqual(test2(), buf)
class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
UnsupportedOperation = pyio.UnsupportedOperation
@staticmethod
def buftype(s):
return s.encode("ascii")
ioclass = io._BytesIO
ioclass = pyio.BytesIO
EOF = b""
def test_read1(self):
@ -333,7 +379,8 @@ class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
self.assertEqual(memio.readinto(b), 0)
self.assertEqual(b, b"")
self.assertRaises(TypeError, memio.readinto, '')
a = array.array(b'b', map(ord, b"hello world"))
import array
a = array.array(b'b', b"hello world")
memio = self.ioclass(buf)
memio.readinto(a)
self.assertEqual(a.tostring(), b"1234567890d")
@ -365,19 +412,41 @@ class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
def test_bytes_array(self):
buf = b"1234567890"
a = array.array(b'b', map(ord, buf))
import array
a = array.array(b'b', buf)
memio = self.ioclass(a)
self.assertEqual(memio.getvalue(), buf)
self.assertEqual(memio.write(a), 10)
self.assertEqual(memio.getvalue(), buf)
class PyStringIOTest(MemoryTestMixin, unittest.TestCase):
class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
buftype = unicode
ioclass = io.StringIO
ioclass = pyio.StringIO
UnsupportedOperation = pyio.UnsupportedOperation
EOF = ""
# TextIO-specific behaviour.
def test_newlines_property(self):
memio = self.ioclass(newline=None)
# The C StringIO decodes newlines in write() calls, but the Python
# implementation only does when reading. This function forces them to
# be decoded for testing.
def force_decode():
memio.seek(0)
memio.read()
self.assertEqual(memio.newlines, None)
memio.write("a\n")
force_decode()
self.assertEqual(memio.newlines, "\n")
memio.write("b\r\n")
force_decode()
self.assertEqual(memio.newlines, ("\n", "\r\n"))
memio.write("c\rd")
force_decode()
self.assertEqual(memio.newlines, ("\r", "\n", "\r\n"))
def test_relative_seek(self):
memio = self.ioclass()
@ -388,29 +457,107 @@ class PyStringIOTest(MemoryTestMixin, unittest.TestCase):
self.assertRaises(IOError, memio.seek, 1, 1)
self.assertRaises(IOError, memio.seek, 1, 2)
def test_textio_properties(self):
memio = self.ioclass()
# These are just dummy values but we nevertheless check them for fear
# of unexpected breakage.
self.assertIsNone(memio.encoding)
self.assertIsNone(memio.errors)
self.assertFalse(memio.line_buffering)
def test_newline_none(self):
# newline=None
memio = self.ioclass("a\nb\r\nc\rd", newline=None)
self.assertEqual(list(memio), ["a\n", "b\n", "c\n", "d"])
memio.seek(0)
self.assertEqual(memio.read(1), "a")
self.assertEqual(memio.read(2), "\nb")
self.assertEqual(memio.read(2), "\nc")
self.assertEqual(memio.read(1), "\n")
memio = self.ioclass(newline=None)
self.assertEqual(2, memio.write("a\n"))
self.assertEqual(3, memio.write("b\r\n"))
self.assertEqual(3, memio.write("c\rd"))
memio.seek(0)
self.assertEqual(memio.read(), "a\nb\nc\nd")
memio = self.ioclass("a\r\nb", newline=None)
self.assertEqual(memio.read(3), "a\nb")
def test_newline_empty(self):
# newline=""
memio = self.ioclass("a\nb\r\nc\rd", newline="")
self.assertEqual(list(memio), ["a\n", "b\r\n", "c\r", "d"])
memio.seek(0)
self.assertEqual(memio.read(4), "a\nb\r")
self.assertEqual(memio.read(2), "\nc")
self.assertEqual(memio.read(1), "\r")
memio = self.ioclass(newline="")
self.assertEqual(2, memio.write("a\n"))
self.assertEqual(2, memio.write("b\r"))
self.assertEqual(2, memio.write("\nc"))
self.assertEqual(2, memio.write("\rd"))
memio.seek(0)
self.assertEqual(list(memio), ["a\n", "b\r\n", "c\r", "d"])
def test_newline_lf(self):
# newline="\n"
memio = self.ioclass("a\nb\r\nc\rd")
self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"])
def test_newline_cr(self):
# newline="\r"
memio = self.ioclass("a\nb\r\nc\rd", newline="\r")
memio.seek(0)
self.assertEqual(memio.read(), "a\rb\r\rc\rd")
memio.seek(0)
self.assertEqual(list(memio), ["a\r", "b\r", "\r", "c\r", "d"])
def test_newline_crlf(self):
# newline="\r\n"
memio = self.ioclass("a\nb\r\nc\rd", newline="\r\n")
memio.seek(0)
self.assertEqual(memio.read(), "a\r\nb\r\r\nc\rd")
memio.seek(0)
self.assertEqual(list(memio), ["a\r\n", "b\r\r\n", "c\rd"])
def test_issue5265(self):
# StringIO can duplicate newlines in universal newlines mode
memio = self.ioclass("a\r\nb\r\n", newline=None)
self.assertEqual(memio.read(5), "a\nb\n")
class CBytesIOTest(PyBytesIOTest):
ioclass = io.BytesIO
UnsupportedOperation = io.UnsupportedOperation
test_bytes_array = unittest.skip(
"array.array() does not have the new buffer API"
)(PyBytesIOTest.test_bytes_array)
class CStringIOTest(PyStringIOTest):
ioclass = io.StringIO
UnsupportedOperation = io.UnsupportedOperation
# XXX: For the Python version of io.StringIO, this is highly
# dependent on the encoding used for the underlying buffer.
# def test_widechar(self):
# buf = self.buftype("\U0002030a\U00020347")
# memio = self.ioclass(buf)
#
# self.assertEqual(memio.getvalue(), buf)
# self.assertEqual(memio.write(buf), len(buf))
# self.assertEqual(memio.tell(), len(buf))
# self.assertEqual(memio.getvalue(), buf)
# self.assertEqual(memio.write(buf), len(buf))
# self.assertEqual(memio.tell(), len(buf) * 2)
# self.assertEqual(memio.getvalue(), buf + buf)
def test_widechar(self):
buf = self.buftype("\U0002030a\U00020347")
memio = self.ioclass(buf)
self.assertEqual(memio.getvalue(), buf)
self.assertEqual(memio.write(buf), len(buf))
self.assertEqual(memio.tell(), len(buf))
self.assertEqual(memio.getvalue(), buf)
self.assertEqual(memio.write(buf), len(buf))
self.assertEqual(memio.tell(), len(buf) * 2)
self.assertEqual(memio.getvalue(), buf + buf)
if has_c_implementation:
class CBytesIOTest(PyBytesIOTest):
ioclass = io.BytesIO
def test_main():
tests = [PyBytesIOTest, PyStringIOTest]
if has_c_implementation:
tests.extend([CBytesIOTest])
test_support.run_unittest(*tests)
tests = [PyBytesIOTest, PyStringIOTest, CBytesIOTest, CStringIOTest]
support.run_unittest(*tests)
if __name__ == '__main__':
test_main()

17
Lib/test/test_support.py
View File

@ -29,7 +29,7 @@ __all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
"run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest",
"BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
"threading_cleanup", "reap_children", "cpython_only",
"check_impl_detail", "get_attribute"]
"check_impl_detail", "get_attribute", "py3k_bytes"]
class Error(Exception):
"""Base class for regression test exceptions."""
@ -968,3 +968,18 @@ def reap_children():
break
except:
break
def py3k_bytes(b):
"""Emulate the py3k bytes() constructor.
NOTE: This is only a best effort function.
"""
try:
# memoryview?
return b.tobytes()
except AttributeError:
try:
# iterable of ints?
return b"".join(chr(x) for x in b)
except TypeError:
return bytes(b)

81
Lib/test/test_univnewlines.py
View File

@ -1,20 +1,25 @@
# Tests universal newline support for both reading and parsing files.
from __future__ import print_function
from __future__ import unicode_literals
import io
import _pyio as pyio
import unittest
import os
import sys
from test import test_support
from test import test_support as support
if not hasattr(sys.stdin, 'newlines'):
raise unittest.SkipTest, \
"This Python does not have universal newline support"
raise unittest.SkipTest(
"This Python does not have universal newline support")
FATX = 'x' * (2**14)
DATA_TEMPLATE = [
"line1=1",
"line2='this is a very long line designed to go past the magic " +
"hundred character limit that is inside fileobject.c and which " +
"is meant to speed up the common case, but we also want to test " +
"line2='this is a very long line designed to go past any default " +
"buffer limits that exist in io.py but we also want to test " +
"the uncommon case, naturally.'",
"def line3():pass",
"line4 = '%s'" % FATX,
@ -28,48 +33,50 @@ DATA_CRLF = "\r\n".join(DATA_TEMPLATE) + "\r\n"
# before end-of-file.
DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r"
DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE]
del x
class TestGenericUnivNewlines(unittest.TestCase):
# use a class variable DATA to define the data to write to the file
# and a class variable NEWLINE to set the expected newlines value
READMODE = 'U'
READMODE = 'r'
WRITEMODE = 'wb'
def setUp(self):
with open(test_support.TESTFN, self.WRITEMODE) as fp:
fp.write(self.DATA)
data = self.DATA
if "b" in self.WRITEMODE:
data = data.encode("ascii")
with self.open(support.TESTFN, self.WRITEMODE) as fp:
fp.write(data)
def tearDown(self):
try:
os.unlink(test_support.TESTFN)
os.unlink(support.TESTFN)
except:
pass
def test_read(self):
with open(test_support.TESTFN, self.READMODE) as fp:
with self.open(support.TESTFN, self.READMODE) as fp:
data = fp.read()
self.assertEqual(data, DATA_LF)
self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
self.assertEqual(set(fp.newlines), set(self.NEWLINE))
def test_readlines(self):
with open(test_support.TESTFN, self.READMODE) as fp:
with self.open(support.TESTFN, self.READMODE) as fp:
data = fp.readlines()
self.assertEqual(data, DATA_SPLIT)
self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
self.assertEqual(set(fp.newlines), set(self.NEWLINE))
def test_readline(self):
with open(test_support.TESTFN, self.READMODE) as fp:
with self.open(support.TESTFN, self.READMODE) as fp:
data = []
d = fp.readline()
while d:
data.append(d)
d = fp.readline()
self.assertEqual(data, DATA_SPLIT)
self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
self.assertEqual(set(fp.newlines), set(self.NEWLINE))
def test_seek(self):
with open(test_support.TESTFN, self.READMODE) as fp:
with self.open(support.TESTFN, self.READMODE) as fp:
fp.readline()
pos = fp.tell()
data = fp.readlines()
@ -78,19 +85,6 @@ class TestGenericUnivNewlines(unittest.TestCase):
data = fp.readlines()
self.assertEqual(data, DATA_SPLIT[1:])
def test_execfile(self):
namespace = {}
execfile(test_support.TESTFN, namespace)
func = namespace['line3']
self.assertEqual(func.func_code.co_firstlineno, 3)
self.assertEqual(namespace['line4'], FATX)
class TestNativeNewlines(TestGenericUnivNewlines):
NEWLINE = None
DATA = DATA_LF
READMODE = 'r'
WRITEMODE = 'w'
class TestCRNewlines(TestGenericUnivNewlines):
NEWLINE = '\r'
@ -105,7 +99,7 @@ class TestCRLFNewlines(TestGenericUnivNewlines):
DATA = DATA_CRLF
def test_tell(self):
with open(test_support.TESTFN, self.READMODE) as fp:
with self.open(support.TESTFN, self.READMODE) as fp:
self.assertEqual(repr(fp.newlines), repr(None))
data = fp.readline()
pos = fp.tell()
@ -117,13 +111,22 @@ class TestMixedNewlines(TestGenericUnivNewlines):
def test_main():
test_support.run_unittest(
TestNativeNewlines,
TestCRNewlines,
TestLFNewlines,
TestCRLFNewlines,
TestMixedNewlines
)
base_tests = (TestCRNewlines,
TestLFNewlines,
TestCRLFNewlines,
TestMixedNewlines)
tests = []
# Test the C and Python implementations.
for test in base_tests:
class CTest(test):
open = io.open
CTest.__name__ = str("C" + test.__name__)
class PyTest(test):
open = staticmethod(pyio.open)
PyTest.__name__ = str("Py" + test.__name__)
tests.append(CTest)
tests.append(PyTest)
support.run_unittest(*tests)
if __name__ == '__main__':
test_main()

4
Misc/NEWS
View File

@ -317,6 +317,10 @@ Core and Builtins
Library
-------
- Issue #6215: All bug fixes and enhancements from the Python 3.1 io library
(including the fast C implementation) have been backported to the standard
``io`` module.
- Issue #6258: Support AMD64 in bdist_msi.
- Issue #5262: Fixed bug in next rollover time computation in

735
Modules/_io/_iomodule.c Normal file
View File

@ -0,0 +1,735 @@
/*
An implementation of the new I/O lib as defined by PEP 3116 - "New I/O"
Classes defined here: UnsupportedOperation, BlockingIOError.
Functions defined here: open().
Mostly written by Amaury Forgeot d'Arc
*/
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "structmember.h"
#include "_iomodule.h"
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif /* HAVE_SYS_TYPES_H */
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif /* HAVE_SYS_STAT_H */
/* Various interned strings */
PyObject *_PyIO_str_close;
PyObject *_PyIO_str_closed;
PyObject *_PyIO_str_decode;
PyObject *_PyIO_str_encode;
PyObject *_PyIO_str_fileno;
PyObject *_PyIO_str_flush;
PyObject *_PyIO_str_getstate;
PyObject *_PyIO_str_isatty;
PyObject *_PyIO_str_newlines;
PyObject *_PyIO_str_nl;
PyObject *_PyIO_str_read;
PyObject *_PyIO_str_read1;
PyObject *_PyIO_str_readable;
PyObject *_PyIO_str_readinto;
PyObject *_PyIO_str_readline;
PyObject *_PyIO_str_reset;
PyObject *_PyIO_str_seek;
PyObject *_PyIO_str_seekable;
PyObject *_PyIO_str_setstate;
PyObject *_PyIO_str_tell;
PyObject *_PyIO_str_truncate;
PyObject *_PyIO_str_writable;
PyObject *_PyIO_str_write;
PyObject *_PyIO_empty_str;
PyObject *_PyIO_empty_bytes;
PyObject *_PyIO_zero;
PyDoc_STRVAR(module_doc,
"The io module provides the Python interfaces to stream handling. The\n"
"builtin open function is defined in this module.\n"
"\n"
"At the top of the I/O hierarchy is the abstract base class IOBase. It\n"
"defines the basic interface to a stream. Note, however, that there is no\n"
"seperation between reading and writing to streams; implementations are\n"
"allowed to throw an IOError if they do not support a given operation.\n"
"\n"
"Extending IOBase is RawIOBase which deals simply with the reading and\n"
"writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide\n"
"an interface to OS files.\n"
"\n"
"BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\n"
"subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\n"
"streams that are readable, writable, and both respectively.\n"
"BufferedRandom provides a buffered interface to random access\n"
"streams. BytesIO is a simple stream of in-memory bytes.\n"
"\n"
"Another IOBase subclass, TextIOBase, deals with the encoding and decoding\n"
"of streams into text. TextIOWrapper, which extends it, is a buffered text\n"
"interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\n"
"is a in-memory stream for text.\n"
"\n"
"Argument names are not part of the specification, and only the arguments\n"
"of open() are intended to be used as keyword arguments.\n"
"\n"
"data:\n"
"\n"
"DEFAULT_BUFFER_SIZE\n"
"\n"
" An int containing the default buffer size used by the module's buffered\n"
" I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n"
" possible.\n"
);
/*
* BlockingIOError extends IOError
*/
static int
blockingioerror_init(PyBlockingIOErrorObject *self, PyObject *args,
PyObject *kwds)
{
PyObject *myerrno = NULL, *strerror = NULL;
PyObject *baseargs = NULL;
Py_ssize_t written = 0;
assert(PyTuple_Check(args));
self->written = 0;
if (!PyArg_ParseTuple(args, "OO|n:BlockingIOError",
&myerrno, &strerror, &written))
return -1;
baseargs = PyTuple_Pack(2, myerrno, strerror);
if (baseargs == NULL)
return -1;
/* This will take care of initializing of myerrno and strerror members */
if (((PyTypeObject *)PyExc_IOError)->tp_init(
(PyObject *)self, baseargs, kwds) == -1) {
Py_DECREF(baseargs);
return -1;
}
Py_DECREF(baseargs);
self->written = written;
return 0;
}
static PyMemberDef blockingioerror_members[] = {
{"characters_written", T_PYSSIZET, offsetof(PyBlockingIOErrorObject, written), 0},
{NULL} /* Sentinel */
};
static PyTypeObject _PyExc_BlockingIOError = {
PyVarObject_HEAD_INIT(NULL, 0)
"BlockingIOError", /*tp_name*/
sizeof(PyBlockingIOErrorObject), /*tp_basicsize*/
0, /*tp_itemsize*/
0, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare */
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
PyDoc_STR("Exception raised when I/O would block "
"on a non-blocking I/O stream"), /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
0, /* tp_methods */
blockingioerror_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)blockingioerror_init, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
};
PyObject *PyExc_BlockingIOError = (PyObject *)&_PyExc_BlockingIOError;
/*
* The main open() function
*/
PyDoc_STRVAR(open_doc,
"Open file and return a stream. Raise IOError upon failure.\n"
"\n"
"file is either a text or byte string giving the name (and the path\n"
"if the file isn't in the current working directory) of the file to\n"
"be opened or an integer file descriptor of the file to be\n"
"wrapped. (If a file descriptor is given, it is closed when the\n"
"returned I/O object is closed, unless closefd is set to False.)\n"
"\n"
"mode is an optional string that specifies the mode in which the file\n"
"is opened. It defaults to 'r' which means open for reading in text\n"
"mode. Other common values are 'w' for writing (truncating the file if\n"
"it already exists), and 'a' for appending (which on some Unix systems,\n"
"means that all writes append to the end of the file regardless of the\n"
"current seek position). In text mode, if encoding is not specified the\n"
"encoding used is platform dependent. (For reading and writing raw\n"
"bytes use binary mode and leave encoding unspecified.) The available\n"
"modes are:\n"
"\n"
"========= ===============================================================\n"
"Character Meaning\n"
"--------- ---------------------------------------------------------------\n"
"'r' open for reading (default)\n"
"'w' open for writing, truncating the file first\n"
"'a' open for writing, appending to the end of the file if it exists\n"
"'b' binary mode\n"
"'t' text mode (default)\n"
"'+' open a disk file for updating (reading and writing)\n"
"'U' universal newline mode (for backwards compatibility; unneeded\n"
" for new code)\n"
"========= ===============================================================\n"
"\n"
"The default mode is 'rt' (open for reading text). For binary random\n"
"access, the mode 'w+b' opens and truncates the file to 0 bytes, while\n"
"'r+b' opens the file without truncation.\n"
"\n"
"Python distinguishes between files opened in binary and text modes,\n"
"even when the underlying operating system doesn't. Files opened in\n"
"binary mode (appending 'b' to the mode argument) return contents as\n"
"bytes objects without any decoding. In text mode (the default, or when\n"
"'t' is appended to the mode argument), the contents of the file are\n"
"returned as strings, the bytes having been first decoded using a\n"
"platform-dependent encoding or using the specified encoding if given.\n"
"\n"
"buffering is an optional integer used to set the buffering policy. By\n"
"default full buffering is on. Pass 0 to switch buffering off (only\n"
"allowed in binary mode), 1 to set line buffering, and an integer > 1\n"
"for full buffering.\n"
"\n"
"encoding is the name of the encoding used to decode or encode the\n"
"file. This should only be used in text mode. The default encoding is\n"
"platform dependent, but any encoding supported by Python can be\n"
"passed. See the codecs module for the list of supported encodings.\n"
"\n"
"errors is an optional string that specifies how encoding errors are to\n"
"be handled---this argument should not be used in binary mode. Pass\n"
"'strict' to raise a ValueError exception if there is an encoding error\n"
"(the default of None has the same effect), or pass 'ignore' to ignore\n"
"errors. (Note that ignoring encoding errors can lead to data loss.)\n"
"See the documentation for codecs.register for a list of the permitted\n"
"encoding error strings.\n"
"\n"
"newline controls how universal newlines works (it only applies to text\n"
"mode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\n"
"follows:\n"
"\n"
"* On input, if newline is None, universal newlines mode is\n"
" enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n"
" these are translated into '\\n' before being returned to the\n"
" caller. If it is '', universal newline mode is enabled, but line\n"
" endings are returned to the caller untranslated. If it has any of\n"
" the other legal values, input lines are only terminated by the given\n"
" string, and the line ending is returned to the caller untranslated.\n"
"\n"
"* On output, if newline is None, any '\\n' characters written are\n"
" translated to the system default line separator, os.linesep. If\n"
" newline is '', no translation takes place. If newline is any of the\n"
" other legal values, any '\\n' characters written are translated to\n"
" the given string.\n"
"\n"
"If closefd is False, the underlying file descriptor will be kept open\n"
"when the file is closed. This does not work when a file name is given\n"
"and must be True in that case.\n"
"\n"
"open() returns a file object whose type depends on the mode, and\n"
"through which the standard file operations such as reading and writing\n"
"are performed. When open() is used to open a file in a text mode ('w',\n"
"'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\n"
"a file in a binary mode, the returned class varies: in read binary\n"
"mode, it returns a BufferedReader; in write binary and append binary\n"
"modes, it returns a BufferedWriter, and in read/write mode, it returns\n"
"a BufferedRandom.\n"
"\n"
"It is also possible to use a string or bytearray as a file for both\n"
"reading and writing. For strings StringIO can be used like a file\n"
"opened in a text mode, and for bytes a BytesIO can be used like a file\n"
"opened in a binary mode.\n"
);
static PyObject *
io_open(PyObject *self, PyObject *args, PyObject *kwds)
{
char *kwlist[] = {"file", "mode", "buffering",
"encoding", "errors", "newline",
"closefd", NULL};
PyObject *file;
char *mode = "r";
int buffering = -1, closefd = 1;
char *encoding = NULL, *errors = NULL, *newline = NULL;
unsigned i;
int reading = 0, writing = 0, appending = 0, updating = 0;
int text = 0, binary = 0, universal = 0;
char rawmode[5], *m;
int line_buffering, isatty;
PyObject *raw, *modeobj = NULL, *buffer = NULL, *wrapper = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist,
&file, &mode, &buffering,
&encoding, &errors, &newline,
&closefd)) {
return NULL;
}
if (!PyUnicode_Check(file) &&
!PyBytes_Check(file) &&
!PyNumber_Check(file)) {
PyObject *repr = PyObject_Repr(file);
if (repr != NULL) {
PyErr_Format(PyExc_TypeError, "invalid file: %s",
PyString_AS_STRING(repr));
Py_DECREF(repr);
}
return NULL;
}
/* Decode mode */
for (i = 0; i < strlen(mode); i++) {
char c = mode[i];
switch (c) {
case 'r':
reading = 1;
break;
case 'w':
writing = 1;
break;
case 'a':
appending = 1;
break;
case '+':
updating = 1;
break;
case 't':
text = 1;
break;
case 'b':
binary = 1;
break;
case 'U':
universal = 1;
reading = 1;
break;
default:
goto invalid_mode;
}
/* c must not be duplicated */
if (strchr(mode+i+1, c)) {
invalid_mode:
PyErr_Format(PyExc_ValueError, "invalid mode: '%s'", mode);
return NULL;
}
}
m = rawmode;
if (reading) *(m++) = 'r';
if (writing) *(m++) = 'w';
if (appending) *(m++) = 'a';
if (updating) *(m++) = '+';
*m = '\0';
/* Parameters validation */
if (universal) {
if (writing || appending) {
PyErr_SetString(PyExc_ValueError,
"can't use U and writing mode at once");
return NULL;
}
reading = 1;
}
if (text && binary) {
PyErr_SetString(PyExc_ValueError,
"can't have text and binary mode at once");
return NULL;
}
if (reading + writing + appending > 1) {
PyErr_SetString(PyExc_ValueError,
"must have exactly one of read/write/append mode");
return NULL;
}
if (binary && encoding != NULL) {
PyErr_SetString(PyExc_ValueError,
"binary mode doesn't take an encoding argument");
return NULL;
}
if (binary && errors != NULL) {
PyErr_SetString(PyExc_ValueError,
"binary mode doesn't take an errors argument");
return NULL;
}
if (binary && newline != NULL) {
PyErr_SetString(PyExc_ValueError,
"binary mode doesn't take a newline argument");
return NULL;
}
/* Create the Raw file stream */
raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type,
"Osi", file, rawmode, closefd);
if (raw == NULL)
return NULL;
modeobj = PyUnicode_FromString(mode);
if (modeobj == NULL)
goto error;
/* buffering */
{
PyObject *res = PyObject_CallMethod(raw, "isatty", NULL);
if (res == NULL)
goto error;
isatty = PyLong_AsLong(res);
Py_DECREF(res);
if (isatty == -1 && PyErr_Occurred())
goto error;
}
if (buffering == 1 || (buffering < 0 && isatty)) {
buffering = -1;
line_buffering = 1;
}
else
line_buffering = 0;
if (buffering < 0) {
buffering = DEFAULT_BUFFER_SIZE;
#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
{
struct stat st;
long fileno;
PyObject *res = PyObject_CallMethod(raw, "fileno", NULL);
if (res == NULL)
goto error;
fileno = PyInt_AsLong(res);
Py_DECREF(res);
if (fileno == -1 && PyErr_Occurred())
goto error;
if (fstat(fileno, &st) >= 0)
buffering = st.st_blksize;
}
#endif
}
if (buffering < 0) {
PyErr_SetString(PyExc_ValueError,
"invalid buffering size");
goto error;
}
/* if not buffering, returns the raw file object */
if (buffering == 0) {
if (!binary) {
PyErr_SetString(PyExc_ValueError,
"can't have unbuffered text I/O");
goto error;
}
Py_DECREF(modeobj);
return raw;
}
/* wraps into a buffered file */
{
PyObject *Buffered_class;
if (updating)
Buffered_class = (PyObject *)&PyBufferedRandom_Type;
else if (writing || appending)
Buffered_class = (PyObject *)&PyBufferedWriter_Type;
else if (reading)
Buffered_class = (PyObject *)&PyBufferedReader_Type;
else {
PyErr_Format(PyExc_ValueError,
"unknown mode: '%s'", mode);
goto error;
}
buffer = PyObject_CallFunction(Buffered_class, "Oi", raw, buffering);
}
Py_CLEAR(raw);
if (buffer == NULL)
goto error;
/* if binary, returns the buffered file */
if (binary) {
Py_DECREF(modeobj);
return buffer;
}
/* wraps into a TextIOWrapper */
wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type,
"Osssi",
buffer,
encoding, errors, newline,
line_buffering);
Py_CLEAR(buffer);
if (wrapper == NULL)
goto error;
if (PyObject_SetAttrString(wrapper, "mode", modeobj) < 0)
goto error;
Py_DECREF(modeobj);
return wrapper;
error:
Py_XDECREF(raw);
Py_XDECREF(modeobj);
Py_XDECREF(buffer);
Py_XDECREF(wrapper);
return NULL;
}
/*
* Private helpers for the io module.
*/
Py_off_t
PyNumber_AsOff_t(PyObject *item, PyObject *err)
{
Py_off_t result;
PyObject *runerr;
PyObject *value = PyNumber_Index(item);
if (value == NULL)
return -1;
if (PyInt_Check(value)) {
/* We assume a long always fits in a Py_off_t... */
result = (Py_off_t) PyInt_AS_LONG(value);
goto finish;
}
/* We're done if PyLong_AsSsize_t() returns without error. */
result = PyLong_AsOff_t(value);
if (result != -1 || !(runerr = PyErr_Occurred()))
goto finish;
/* Error handling code -- only manage OverflowError differently */
if (!PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError))
goto finish;
PyErr_Clear();
/* If no error-handling desired then the default clipping
is sufficient.
*/
if (!err) {
assert(PyLong_Check(value));
/* Whether or not it is less than or equal to
zero is determined by the sign of ob_size
*/
if (_PyLong_Sign(value) < 0)
result = PY_OFF_T_MIN;
else
result = PY_OFF_T_MAX;
}
else {
/* Otherwise replace the error with caller's error object. */
PyErr_Format(err,
"cannot fit '%.200s' into an offset-sized integer",
item->ob_type->tp_name);
}
finish:
Py_DECREF(value);
return result;
}
/*
* Module definition
*/
PyObject *_PyIO_os_module = NULL;
PyObject *_PyIO_locale_module = NULL;
PyObject *_PyIO_unsupported_operation = NULL;
static PyMethodDef module_methods[] = {
{"open", (PyCFunction)io_open, METH_VARARGS|METH_KEYWORDS, open_doc},
{NULL, NULL}
};
PyMODINIT_FUNC
init_io(void)
{
PyObject *m = Py_InitModule4("_io", module_methods,
module_doc, NULL, PYTHON_API_VERSION);
if (m == NULL)
return;
/* put os in the module state */
_PyIO_os_module = PyImport_ImportModule("os");
if (_PyIO_os_module == NULL)
goto fail;
#define ADD_TYPE(type, name) \
if (PyType_Ready(type) < 0) \
goto fail; \
Py_INCREF(type); \
if (PyModule_AddObject(m, name, (PyObject *)type) < 0) { \
Py_DECREF(type); \
goto fail; \
}
/* DEFAULT_BUFFER_SIZE */
if (PyModule_AddIntMacro(m, DEFAULT_BUFFER_SIZE) < 0)
goto fail;
/* UnsupportedOperation inherits from ValueError and IOError */
_PyIO_unsupported_operation = PyObject_CallFunction(
(PyObject *)&PyType_Type, "s(OO){}",
"UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
if (_PyIO_unsupported_operation == NULL)
goto fail;
Py_INCREF(_PyIO_unsupported_operation);
if (PyModule_AddObject(m, "UnsupportedOperation",
_PyIO_unsupported_operation) < 0)
goto fail;
/* BlockingIOError */
_PyExc_BlockingIOError.tp_base = (PyTypeObject *) PyExc_IOError;
ADD_TYPE(&_PyExc_BlockingIOError, "BlockingIOError");
/* Concrete base types of the IO ABCs.
(the ABCs themselves are declared through inheritance in io.py)
*/
ADD_TYPE(&PyIOBase_Type, "_IOBase");
ADD_TYPE(&PyRawIOBase_Type, "_RawIOBase");
ADD_TYPE(&PyBufferedIOBase_Type, "_BufferedIOBase");
ADD_TYPE(&PyTextIOBase_Type, "_TextIOBase");
/* Implementation of concrete IO objects. */
/* FileIO */
PyFileIO_Type.tp_base = &PyRawIOBase_Type;
ADD_TYPE(&PyFileIO_Type, "FileIO");
/* BytesIO */
PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBytesIO_Type, "BytesIO");
/* StringIO */
PyStringIO_Type.tp_base = &PyTextIOBase_Type;
ADD_TYPE(&PyStringIO_Type, "StringIO");
/* BufferedReader */
PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBufferedReader_Type, "BufferedReader");
/* BufferedWriter */
PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBufferedWriter_Type, "BufferedWriter");
/* BufferedRWPair */
PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBufferedRWPair_Type, "BufferedRWPair");
/* BufferedRandom */
PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type;
ADD_TYPE(&PyBufferedRandom_Type, "BufferedRandom");
/* TextIOWrapper */
PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type;
ADD_TYPE(&PyTextIOWrapper_Type, "TextIOWrapper");
/* IncrementalNewlineDecoder */
ADD_TYPE(&PyIncrementalNewlineDecoder_Type, "IncrementalNewlineDecoder");
/* Interned strings */
if (!(_PyIO_str_close = PyString_InternFromString("close")))
goto fail;
if (!(_PyIO_str_closed = PyString_InternFromString("closed")))
goto fail;
if (!(_PyIO_str_decode = PyString_InternFromString("decode")))
goto fail;
if (!(_PyIO_str_encode = PyString_InternFromString("encode")))
goto fail;
if (!(_PyIO_str_fileno = PyString_InternFromString("fileno")))
goto fail;
if (!(_PyIO_str_flush = PyString_InternFromString("flush")))
goto fail;
if (!(_PyIO_str_getstate = PyString_InternFromString("getstate")))
goto fail;
if (!(_PyIO_str_isatty = PyString_InternFromString("isatty")))
goto fail;
if (!(_PyIO_str_newlines = PyString_InternFromString("newlines")))
goto fail;
if (!(_PyIO_str_nl = PyString_InternFromString("\n")))
goto fail;
if (!(_PyIO_str_read = PyString_InternFromString("read")))
goto fail;
if (!(_PyIO_str_read1 = PyString_InternFromString("read1")))
goto fail;
if (!(_PyIO_str_readable = PyString_InternFromString("readable")))
goto fail;
if (!(_PyIO_str_readinto = PyString_InternFromString("readinto")))
goto fail;
if (!(_PyIO_str_readline = PyString_InternFromString("readline")))
goto fail;
if (!(_PyIO_str_reset = PyString_InternFromString("reset")))
goto fail;
if (!(_PyIO_str_seek = PyString_InternFromString("seek")))
goto fail;
if (!(_PyIO_str_seekable = PyString_InternFromString("seekable")))
goto fail;
if (!(_PyIO_str_setstate = PyString_InternFromString("setstate")))
goto fail;
if (!(_PyIO_str_tell = PyString_InternFromString("tell")))
goto fail;
if (!(_PyIO_str_truncate = PyString_InternFromString("truncate")))
goto fail;
if (!(_PyIO_str_write = PyString_InternFromString("write")))
goto fail;
if (!(_PyIO_str_writable = PyString_InternFromString("writable")))
goto fail;
if (!(_PyIO_empty_str = PyUnicode_FromStringAndSize(NULL, 0)))
goto fail;
if (!(_PyIO_empty_bytes = PyBytes_FromStringAndSize(NULL, 0)))
goto fail;
if (!(_PyIO_zero = PyLong_FromLong(0L)))
goto fail;
return;
fail:
Py_CLEAR(_PyIO_os_module);
Py_CLEAR(_PyIO_unsupported_operation);
Py_DECREF(m);
}

146
Modules/_io/_iomodule.h Normal file
View File

@ -0,0 +1,146 @@
/*
* Declarations shared between the different parts of the io module
*/
/* ABCs */
extern PyTypeObject PyIOBase_Type;
extern PyTypeObject PyRawIOBase_Type;
extern PyTypeObject PyBufferedIOBase_Type;
extern PyTypeObject PyTextIOBase_Type;
/* Concrete classes */
extern PyTypeObject PyFileIO_Type;
extern PyTypeObject PyBytesIO_Type;
extern PyTypeObject PyStringIO_Type;
extern PyTypeObject PyBufferedReader_Type;
extern PyTypeObject PyBufferedWriter_Type;
extern PyTypeObject PyBufferedRWPair_Type;
extern PyTypeObject PyBufferedRandom_Type;
extern PyTypeObject PyTextIOWrapper_Type;
extern PyTypeObject PyIncrementalNewlineDecoder_Type;
/* These functions are used as METH_NOARGS methods, are normally called
* with args=NULL, and return a new reference.
* BUT when args=Py_True is passed, they return a borrowed reference.
*/
extern PyObject* _PyIOBase_check_readable(PyObject *self, PyObject *args);
extern PyObject* _PyIOBase_check_writable(PyObject *self, PyObject *args);
extern PyObject* _PyIOBase_check_seekable(PyObject *self, PyObject *args);
extern PyObject* _PyIOBase_check_closed(PyObject *self, PyObject *args);
/* Helper for finalization.
This function will revive an object ready to be deallocated and try to
close() it. It returns 0 if the object can be destroyed, or -1 if it
is alive again. */
extern int _PyIOBase_finalize(PyObject *self);
/* Returns true if the given FileIO object is closed.
Doesn't check the argument type, so be careful! */
extern int _PyFileIO_closed(PyObject *self);
/* Shortcut to the core of the IncrementalNewlineDecoder.decode method */
extern PyObject *_PyIncrementalNewlineDecoder_decode(
PyObject *self, PyObject *input, int final);
/* Finds the first line ending between `start` and `end`.
If found, returns the index after the line ending and doesn't touch
`*consumed`.
If not found, returns -1 and sets `*consumed` to the number of characters
which can be safely put aside until another search.
NOTE: for performance reasons, `end` must point to a NUL character ('\0').
Otherwise, the function will scan further and return garbage. */
extern Py_ssize_t _PyIO_find_line_ending(
int translated, int universal, PyObject *readnl,
Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed);
#define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */
typedef struct {
/* This is the equivalent of PyException_HEAD in 3.x */
PyObject_HEAD
PyObject *dict;
PyObject *args;
PyObject *message;
PyObject *myerrno;
PyObject *strerror;
PyObject *filename; /* Not used, but part of the IOError object */
Py_ssize_t written;
} PyBlockingIOErrorObject;
PyAPI_DATA(PyObject *) PyExc_BlockingIOError;
/*
* Offset type for positioning.
*/
#if defined(MS_WIN64) || defined(MS_WINDOWS)
/* Windows uses long long for offsets */
typedef PY_LONG_LONG Py_off_t;
# define PyLong_AsOff_t PyLong_AsLongLong
# define PyLong_FromOff_t PyLong_FromLongLong
# define PY_OFF_T_MAX PY_LLONG_MAX
# define PY_OFF_T_MIN PY_LLONG_MIN
#else
/* Other platforms use off_t */
typedef off_t Py_off_t;
#if (SIZEOF_OFF_T == SIZEOF_SIZE_T)
# define PyLong_AsOff_t PyLong_AsSsize_t
# define PyLong_FromOff_t PyLong_FromSsize_t
# define PY_OFF_T_MAX PY_SSIZE_T_MAX
# define PY_OFF_T_MIN PY_SSIZE_T_MIN
#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG)
# define PyLong_AsOff_t PyLong_AsLongLong
# define PyLong_FromOff_t PyLong_FromLongLong
# define PY_OFF_T_MAX PY_LLONG_MAX
# define PY_OFF_T_MIN PY_LLONG_MIN
#elif (SIZEOF_OFF_T == SIZEOF_LONG)
# define PyLong_AsOff_t PyLong_AsLong
# define PyLong_FromOff_t PyLong_FromLong
# define PY_OFF_T_MAX LONG_MAX
# define PY_OFF_T_MIN LONG_MIN
#else
# error off_t does not match either size_t, long, or long long!
#endif
#endif
extern Py_off_t PyNumber_AsOff_t(PyObject *item, PyObject *err);
/* Implementation details */
extern PyObject *_PyIO_os_module;
extern PyObject *_PyIO_locale_module;
extern PyObject *_PyIO_unsupported_operation;
extern PyObject *_PyIO_str_close;
extern PyObject *_PyIO_str_closed;
extern PyObject *_PyIO_str_decode;
extern PyObject *_PyIO_str_encode;
extern PyObject *_PyIO_str_fileno;
extern PyObject *_PyIO_str_flush;
extern PyObject *_PyIO_str_getstate;
extern PyObject *_PyIO_str_isatty;
extern PyObject *_PyIO_str_newlines;
extern PyObject *_PyIO_str_nl;
extern PyObject *_PyIO_str_read;
extern PyObject *_PyIO_str_read1;
extern PyObject *_PyIO_str_readable;
extern PyObject *_PyIO_str_readinto;
extern PyObject *_PyIO_str_readline;
extern PyObject *_PyIO_str_reset;
extern PyObject *_PyIO_str_seek;
extern PyObject *_PyIO_str_seekable;
extern PyObject *_PyIO_str_setstate;
extern PyObject *_PyIO_str_tell;
extern PyObject *_PyIO_str_truncate;
extern PyObject *_PyIO_str_writable;
extern PyObject *_PyIO_str_write;
extern PyObject *_PyIO_empty_str;
extern PyObject *_PyIO_empty_bytes;
extern PyObject *_PyIO_zero;

2289
Modules/_io/bufferedio.c Normal file
View File

File diff suppressed because it is too large Load Diff

201
Modules/_bytesio.c → Modules/_io/bytesio.c
View File

@ -1,4 +1,6 @@
#include "Python.h"
#include "structmember.h" /* for offsetof() */
#include "_iomodule.h"
typedef struct {
PyObject_HEAD
@ -6,7 +8,9 @@ typedef struct {
Py_ssize_t pos;
Py_ssize_t string_size;
size_t buf_size;
} BytesIOObject;
PyObject *dict;
PyObject *weakreflist;
} bytesio;
#define CHECK_CLOSED(self) \
if ((self)->buf == NULL) { \
@ -19,7 +23,7 @@ typedef struct {
object. Returns the length between the current position to the
next newline character. */
static Py_ssize_t
get_line(BytesIOObject *self, char **output)
get_line(bytesio *self, char **output)
{
char *n;
const char *str_end;
@ -52,7 +56,7 @@ get_line(BytesIOObject *self, char **output)
The caller should ensure that the 'size' argument is non-negative. Returns
0 on success, -1 otherwise. */
static int
resize_buffer(BytesIOObject *self, size_t size)
resize_buffer(bytesio *self, size_t size)
{
/* Here, unsigned types are used to avoid dealing with signed integer
overflow, which is undefined in C. */
@ -104,7 +108,7 @@ resize_buffer(BytesIOObject *self, size_t size)
/* Internal routine for writing a string of bytes to the buffer of a BytesIO
object. Returns the number of bytes wrote, or -1 on error. */
static Py_ssize_t
write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len)
write_bytes(bytesio *self, const char *bytes, Py_ssize_t len)
{
assert(self->buf != NULL);
assert(self->pos >= 0);
@ -142,17 +146,19 @@ write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len)
}
static PyObject *
bytesio_get_closed(BytesIOObject *self)
bytesio_get_closed(bytesio *self)
{
if (self->buf == NULL)
if (self->buf == NULL) {
Py_RETURN_TRUE;
else
}
else {
Py_RETURN_FALSE;
}
}
/* Generic getter for the writable, readable and seekable properties */
static PyObject *
return_true(BytesIOObject *self)
return_true(bytesio *self)
{
Py_RETURN_TRUE;
}
@ -161,7 +167,7 @@ PyDoc_STRVAR(flush_doc,
"flush() -> None. Does nothing.");
static PyObject *
bytesio_flush(BytesIOObject *self)
bytesio_flush(bytesio *self)
{
Py_RETURN_NONE;
}
@ -172,10 +178,10 @@ PyDoc_STRVAR(getval_doc,
"Retrieve the entire contents of the BytesIO object.");
static PyObject *
bytesio_getvalue(BytesIOObject *self)
bytesio_getvalue(bytesio *self)
{
CHECK_CLOSED(self);
return PyString_FromStringAndSize(self->buf, self->string_size);
return PyBytes_FromStringAndSize(self->buf, self->string_size);
}
PyDoc_STRVAR(isatty_doc,
@ -185,7 +191,7 @@ PyDoc_STRVAR(isatty_doc,
"to a tty-like device.");
static PyObject *
bytesio_isatty(BytesIOObject *self)
bytesio_isatty(bytesio *self)
{
CHECK_CLOSED(self);
Py_RETURN_FALSE;
@ -195,10 +201,10 @@ PyDoc_STRVAR(tell_doc,
"tell() -> current file position, an integer\n");
static PyObject *
bytesio_tell(BytesIOObject *self)
bytesio_tell(bytesio *self)
{
CHECK_CLOSED(self);
return PyInt_FromSsize_t(self->pos);
return PyLong_FromSsize_t(self->pos);
}
PyDoc_STRVAR(read_doc,
@ -208,7 +214,7 @@ PyDoc_STRVAR(read_doc,
"Return an empty string at EOF.");
static PyObject *
bytesio_read(BytesIOObject *self, PyObject *args)
bytesio_read(bytesio *self, PyObject *args)
{
Py_ssize_t size, n;
char *output;
@ -219,8 +225,8 @@ bytesio_read(BytesIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "|O:read", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
if (PyNumber_Check(arg)) {
size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred())
return NULL;
}
@ -246,7 +252,7 @@ bytesio_read(BytesIOObject *self, PyObject *args)
output = self->buf + self->pos;
self->pos += size;
return PyString_FromStringAndSize(output, size);
return PyBytes_FromStringAndSize(output, size);
}
@ -257,7 +263,7 @@ PyDoc_STRVAR(read1_doc,
"Return an empty string at EOF.");
static PyObject *
bytesio_read1(BytesIOObject *self, PyObject *n)
bytesio_read1(bytesio *self, PyObject *n)
{
PyObject *arg, *res;
@ -277,7 +283,7 @@ PyDoc_STRVAR(readline_doc,
"Return an empty string at EOF.\n");
static PyObject *
bytesio_readline(BytesIOObject *self, PyObject *args)
bytesio_readline(bytesio *self, PyObject *args)
{
Py_ssize_t size, n;
char *output;
@ -288,8 +294,8 @@ bytesio_readline(BytesIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "|O:readline", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
if (PyNumber_Check(arg)) {
size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred())
return NULL;
}
@ -311,7 +317,7 @@ bytesio_readline(BytesIOObject *self, PyObject *args)
self->pos -= size;
}
return PyString_FromStringAndSize(output, n);
return PyBytes_FromStringAndSize(output, n);
}
PyDoc_STRVAR(readlines_doc,
@ -322,7 +328,7 @@ PyDoc_STRVAR(readlines_doc,
"total number of bytes in the lines returned.\n");
static PyObject *
bytesio_readlines(BytesIOObject *self, PyObject *args)
bytesio_readlines(bytesio *self, PyObject *args)
{
Py_ssize_t maxsize, size, n;
PyObject *result, *line;
@ -334,8 +340,8 @@ bytesio_readlines(BytesIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "|O:readlines", &arg))
return NULL;
if (PyInt_Check(arg)) {
maxsize = PyInt_AsSsize_t(arg);
if (PyNumber_Check(arg)) {
maxsize = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (maxsize == -1 && PyErr_Occurred())
return NULL;
}
@ -355,7 +361,7 @@ bytesio_readlines(BytesIOObject *self, PyObject *args)
return NULL;
while ((n = get_line(self, &output)) != 0) {
line = PyString_FromStringAndSize(output, n);
line = PyBytes_FromStringAndSize(output, n);
if (!line)
goto on_error;
if (PyList_Append(result, line) == -1) {
@ -381,25 +387,27 @@ PyDoc_STRVAR(readinto_doc,
"is set not to block as has no data to read.");
static PyObject *
bytesio_readinto(BytesIOObject *self, PyObject *buffer)
bytesio_readinto(bytesio *self, PyObject *args)
{
void *raw_buffer;
Py_buffer buf;
Py_ssize_t len;
CHECK_CLOSED(self);
if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1)
if (!PyArg_ParseTuple(args, "w*", &buf))
return NULL;
len = buf.len;
if (self->pos + len > self->string_size)
len = self->string_size - self->pos;
memcpy(raw_buffer, self->buf + self->pos, len);
memcpy(buf.buf, self->buf + self->pos, len);
assert(self->pos + len < PY_SSIZE_T_MAX);
assert(len >= 0);
self->pos += len;
return PyInt_FromSsize_t(len);
PyBuffer_Release(&buf);
return PyLong_FromSsize_t(len);
}
PyDoc_STRVAR(truncate_doc,
@ -409,7 +417,7 @@ PyDoc_STRVAR(truncate_doc,
"Returns the new size. Imply an absolute seek to the position size.");
static PyObject *
bytesio_truncate(BytesIOObject *self, PyObject *args)
bytesio_truncate(bytesio *self, PyObject *args)
{
Py_ssize_t size;
PyObject *arg = Py_None;
@ -419,8 +427,8 @@ bytesio_truncate(BytesIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "|O:truncate", &arg))
return NULL;
if (PyInt_Check(arg)) {
size = PyInt_AsSsize_t(arg);
if (PyNumber_Check(arg)) {
size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred())
return NULL;
}
@ -447,11 +455,11 @@ bytesio_truncate(BytesIOObject *self, PyObject *args)
}
self->pos = size;
return PyInt_FromSsize_t(size);
return PyLong_FromSsize_t(size);
}
static PyObject *
bytesio_iternext(BytesIOObject *self)
bytesio_iternext(bytesio *self)
{
char *next;
Py_ssize_t n;
@ -463,7 +471,7 @@ bytesio_iternext(BytesIOObject *self)
if (!next || n == 0)
return NULL;
return PyString_FromStringAndSize(next, n);
return PyBytes_FromStringAndSize(next, n);
}
PyDoc_STRVAR(seek_doc,
@ -476,27 +484,21 @@ PyDoc_STRVAR(seek_doc,
"Returns the new absolute position.");
static PyObject *
bytesio_seek(BytesIOObject *self, PyObject *args)
bytesio_seek(bytesio *self, PyObject *args)
{
PyObject *pos_obj, *mode_obj;
PyObject *posobj;
Py_ssize_t pos;
int mode = 0;
CHECK_CLOSED(self);
/* Special-case for 2.x to prevent floats from passing through.
This only needed to make a test in test_io succeed. */
if (!PyArg_UnpackTuple(args, "seek", 1, 2, &pos_obj, &mode_obj))
return NULL;
if (PyFloat_Check(pos_obj)) {
PyErr_SetString(PyExc_TypeError,
"position argument must be an integer");
return NULL;
}
if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode))
if (!PyArg_ParseTuple(args, "O|i:seek", &posobj, &mode))
return NULL;
pos = PyNumber_AsSsize_t(posobj, PyExc_OverflowError);
if (pos == -1 && PyErr_Occurred())
return NULL;
if (pos < 0 && mode == 0) {
PyErr_Format(PyExc_ValueError,
"negative seek value %zd", pos);
@ -532,7 +534,7 @@ bytesio_seek(BytesIOObject *self, PyObject *args)
pos = 0;
self->pos = pos;
return PyInt_FromSsize_t(self->pos);
return PyLong_FromSsize_t(self->pos);
}
PyDoc_STRVAR(write_doc,
@ -541,31 +543,24 @@ PyDoc_STRVAR(write_doc,
"Return the number of bytes written.");
static PyObject *
bytesio_write(BytesIOObject *self, PyObject *obj)
bytesio_write(bytesio *self, PyObject *obj)
{
const char *bytes;
Py_ssize_t size;
Py_ssize_t n = 0;
Py_buffer buf;
PyObject *result = NULL;
CHECK_CLOSED(self);
/* Special-case in 2.x to prevent unicode objects to pass through. */
if (PyUnicode_Check(obj)) {
PyErr_SetString(PyExc_TypeError,
"expecting a bytes object, got unicode");
return NULL;
}
if (PyObject_AsReadBuffer(obj, (void *)&bytes, &size) < 0)
if (PyObject_GetBuffer(obj, &buf, PyBUF_CONTIG_RO) < 0)
return NULL;
if (size != 0) {
n = write_bytes(self, bytes, size);
if (n < 0)
return NULL;
}
if (buf.len != 0)
n = write_bytes(self, buf.buf, buf.len);
if (n >= 0)
result = PyLong_FromSsize_t(n);
return PyInt_FromSsize_t(n);
PyBuffer_Release(&buf);
return result;
}
PyDoc_STRVAR(writelines_doc,
@ -576,7 +571,7 @@ PyDoc_STRVAR(writelines_doc,
"each string.");
static PyObject *
bytesio_writelines(BytesIOObject *self, PyObject *v)
bytesio_writelines(bytesio *self, PyObject *v)
{
PyObject *it, *item;
PyObject *ret;
@ -609,7 +604,7 @@ PyDoc_STRVAR(close_doc,
"close() -> None. Disable all I/O operations.");
static PyObject *
bytesio_close(BytesIOObject *self)
bytesio_close(bytesio *self)
{
if (self->buf != NULL) {
PyMem_Free(self->buf);
@ -619,22 +614,23 @@ bytesio_close(BytesIOObject *self)
}
static void
bytesio_dealloc(BytesIOObject *self)
bytesio_dealloc(bytesio *self)
{
if (self->buf != NULL) {
PyMem_Free(self->buf);
self->buf = NULL;
}
Py_TYPE(self)->tp_clear((PyObject *)self);
Py_TYPE(self)->tp_free(self);
}
static PyObject *
bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
BytesIOObject *self;
bytesio *self;
assert(type != NULL && type->tp_alloc != NULL);
self = (BytesIOObject *)type->tp_alloc(type, 0);
self = (bytesio *)type->tp_alloc(type, 0);
if (self == NULL)
return NULL;
@ -651,7 +647,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
}
static int
bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds)
bytesio_init(bytesio *self, PyObject *args, PyObject *kwds)
{
PyObject *initvalue = NULL;
@ -674,10 +670,28 @@ bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds)
return 0;
}
static int
bytesio_traverse(bytesio *self, visitproc visit, void *arg)
{
Py_VISIT(self->dict);
Py_VISIT(self->weakreflist);
return 0;
}
static int
bytesio_clear(bytesio *self)
{
Py_CLEAR(self->dict);
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *)self);
return 0;
}
static PyGetSetDef bytesio_getsetlist[] = {
{"closed", (getter)bytesio_get_closed, NULL,
"True if the file is closed."},
{0}, /* sentinel */
{NULL}, /* sentinel */
};
static struct PyMethodDef bytesio_methods[] = {
@ -691,7 +705,7 @@ static struct PyMethodDef bytesio_methods[] = {
{"write", (PyCFunction)bytesio_write, METH_O, write_doc},
{"writelines", (PyCFunction)bytesio_writelines, METH_O, writelines_doc},
{"read1", (PyCFunction)bytesio_read1, METH_O, read1_doc},
{"readinto", (PyCFunction)bytesio_readinto, METH_O, readinto_doc},
{"readinto", (PyCFunction)bytesio_readinto, METH_VARARGS, readinto_doc},
{"readline", (PyCFunction)bytesio_readline, METH_VARARGS, readline_doc},
{"readlines", (PyCFunction)bytesio_readlines, METH_VARARGS, readlines_doc},
{"read", (PyCFunction)bytesio_read, METH_VARARGS, read_doc},
@ -707,16 +721,16 @@ PyDoc_STRVAR(bytesio_doc,
"Create a buffered I/O implementation using an in-memory bytes\n"
"buffer, ready for reading and writing.");
static PyTypeObject BytesIO_Type = {
PyTypeObject PyBytesIO_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_bytesio._BytesIO", /*tp_name*/
sizeof(BytesIOObject), /*tp_basicsize*/
"_io.BytesIO", /*tp_name*/
sizeof(bytesio), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)bytesio_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_reserved*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
@ -727,12 +741,13 @@ static PyTypeObject BytesIO_Type = {
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_GC, /*tp_flags*/
bytesio_doc, /*tp_doc*/
0, /*tp_traverse*/
0, /*tp_clear*/
(traverseproc)bytesio_traverse, /*tp_traverse*/
(inquiry)bytesio_clear, /*tp_clear*/
0, /*tp_richcompare*/
0, /*tp_weaklistoffset*/
offsetof(bytesio, weakreflist), /*tp_weaklistoffset*/
PyObject_SelfIter, /*tp_iter*/
(iternextfunc)bytesio_iternext, /*tp_iternext*/
bytesio_methods, /*tp_methods*/
@ -742,22 +757,8 @@ static PyTypeObject BytesIO_Type = {
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
0, /*tp_dictoffset*/
offsetof(bytesio, dict), /*tp_dictoffset*/
(initproc)bytesio_init, /*tp_init*/
0, /*tp_alloc*/
bytesio_new, /*tp_new*/
};
PyMODINIT_FUNC
init_bytesio(void)
{
PyObject *m;
if (PyType_Ready(&BytesIO_Type) < 0)
return;
m = Py_InitModule("_bytesio", NULL);
if (m == NULL)
return;
Py_INCREF(&BytesIO_Type);
PyModule_AddObject(m, "_BytesIO", (PyObject *)&BytesIO_Type);
}

419
Modules/_fileio.c → Modules/_io/fileio.c
View File

@ -6,6 +6,7 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <stddef.h> /* For offsetof */
#include "_iomodule.h"
/*
* Known likely problems:
@ -27,6 +28,20 @@
#include <windows.h>
#endif
#if BUFSIZ < (8*1024)
#define SMALLCHUNK (8*1024)
#elif (BUFSIZ >= (2 << 25))
#error "unreasonable BUFSIZ > 64MB defined"
#else
#define SMALLCHUNK BUFSIZ
#endif
#if SIZEOF_INT < 4
#define BIGCHUNK (512 * 32)
#else
#define BIGCHUNK (512 * 1024)
#endif
typedef struct {
PyObject_HEAD
int fd;
@ -35,55 +50,76 @@ typedef struct {
int seekable : 2; /* -1 means unknown */
int closefd : 1;
PyObject *weakreflist;
} PyFileIOObject;
PyObject *dict;
} fileio;
PyTypeObject PyFileIO_Type;
#define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type))
int
_PyFileIO_closed(PyObject *self)
{
return ((fileio *)self)->fd < 0;
}
static PyObject *
portable_lseek(int fd, PyObject *posobj, int whence);
/* Returns 0 on success, errno (which is < 0) on failure. */
static PyObject *portable_lseek(int fd, PyObject *posobj, int whence);
/* Returns 0 on success, -1 with exception set on failure. */
static int
internal_close(PyFileIOObject *self)
internal_close(fileio *self)
{
int err = 0;
int save_errno = 0;
if (self->fd >= 0) {
int fd = self->fd;
self->fd = -1;
Py_BEGIN_ALLOW_THREADS
if (close(fd) < 0)
/* fd is accessible and someone else may have closed it */
if (_PyVerify_fd(fd)) {
Py_BEGIN_ALLOW_THREADS
err = close(fd);
if (err < 0)
save_errno = errno;
Py_END_ALLOW_THREADS
} else {
save_errno = errno;
Py_END_ALLOW_THREADS
err = -1;
}
}
return save_errno;
if (err < 0) {
errno = save_errno;
PyErr_SetFromErrno(PyExc_IOError);
return -1;
}
return 0;
}
static PyObject *
fileio_close(PyFileIOObject *self)
fileio_close(fileio *self)
{
if (!self->closefd) {
self->fd = -1;
Py_RETURN_NONE;
}
errno = internal_close(self);
if (errno < 0) {
PyErr_SetFromErrno(PyExc_IOError);
if (errno < 0)
return NULL;
}
Py_RETURN_NONE;
return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type,
"close", "O", self);
}
static PyObject *
fileio_new(PyTypeObject *type, PyObject *args, PyObject *kews)
fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
PyFileIOObject *self;
fileio *self;
assert(type != NULL && type->tp_alloc != NULL);
self = (PyFileIOObject *) type->tp_alloc(type, 0);
self = (fileio *) type->tp_alloc(type, 0);
if (self != NULL) {
self->fd = -1;
self->readable = 0;
@ -101,7 +137,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kews)
directories, so we need a check. */
static int
dircheck(PyFileIOObject* self, char *name)
dircheck(fileio* self, const char *name)
{
#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
struct stat buf;
@ -110,7 +146,8 @@ dircheck(PyFileIOObject* self, char *name)
if (fstat(self->fd, &buf) == 0 && S_ISDIR(buf.st_mode)) {
char *msg = strerror(EISDIR);
PyObject *exc;
internal_close(self);
if (internal_close(self))
return -1;
exc = PyObject_CallFunction(PyExc_IOError, "(iss)",
EISDIR, msg, name);
@ -144,9 +181,10 @@ check_fd(int fd)
static int
fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
{
PyFileIOObject *self = (PyFileIOObject *) oself;
fileio *self = (fileio *) oself;
static char *kwlist[] = {"file", "mode", "closefd", NULL};
char *name = NULL;
const char *name = NULL;
PyObject *nameobj, *stringobj = NULL;
char *mode = "r";
char *s;
#ifdef MS_WINDOWS
@ -165,42 +203,59 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
return -1;
}
if (PyArg_ParseTupleAndKeywords(args, kwds, "i|si:fileio",
kwlist, &fd, &mode, &closefd)) {
if (fd < 0) {
if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio",
kwlist, &nameobj, &mode, &closefd))
return -1;
if (PyFloat_Check(nameobj)) {
PyErr_SetString(PyExc_TypeError,
"integer argument expected, got float");
return -1;
}
fd = PyLong_AsLong(nameobj);
if (fd < 0) {
if (!PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
"Negative filedescriptor");
return -1;
}
if (check_fd(fd))
return -1;
}
else {
PyErr_Clear();
}
#ifdef MS_WINDOWS
if (GetVersion() < 0x80000000) {
if (GetVersion() < 0x80000000) {
/* On NT, so wide API available */
PyObject *po;
if (PyArg_ParseTupleAndKeywords(args, kwds, "U|si:fileio",
kwlist, &po, &mode, &closefd)
) {
widename = PyUnicode_AS_UNICODE(po);
} else {
/* Drop the argument parsing error as narrow
strings are also valid. */
PyErr_Clear();
}
}
if (widename == NULL)
if (PyUnicode_Check(nameobj))
widename = PyUnicode_AS_UNICODE(nameobj);
}
if (widename == NULL)
#endif
{
if (!PyArg_ParseTupleAndKeywords(args, kwds, "et|si:fileio",
kwlist,
Py_FileSystemDefaultEncoding,
&name, &mode, &closefd))
return -1;
}
if (fd < 0)
{
if (PyBytes_Check(nameobj) || PyByteArray_Check(nameobj)) {
Py_ssize_t namelen;
if (PyObject_AsCharBuffer(nameobj, &name, &namelen) < 0)
return -1;
}
else {
PyObject *u = PyUnicode_FromObject(nameobj);
if (u == NULL)
return -1;
stringobj = PyUnicode_AsEncodedString(
u, Py_FileSystemDefaultEncoding, "surrogateescape");
Py_DECREF(u);
if (stringobj == NULL)
return -1;
if (!PyBytes_Check(stringobj)) {
PyErr_SetString(PyExc_TypeError,
"encoder failed to return bytes");
goto error;
}
name = PyBytes_AS_STRING(stringobj);
}
}
s = mode;
@ -266,6 +321,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
#endif
if (fd >= 0) {
if (check_fd(fd))
goto error;
self->fd = fd;
self->closefd = closefd;
}
@ -299,6 +356,9 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
goto error;
}
if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0)
goto error;
if (append) {
/* For consistent behaviour, we explicitly seek to the
end of file (otherwise, it might be done only on the
@ -315,24 +375,33 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
ret = -1;
done:
PyMem_Free(name);
Py_CLEAR(stringobj);
return ret;
}
static void
fileio_dealloc(PyFileIOObject *self)
static int
fileio_traverse(fileio *self, visitproc visit, void *arg)
{
Py_VISIT(self->dict);
return 0;
}
static int
fileio_clear(fileio *self)
{
Py_CLEAR(self->dict);
return 0;
}
static void
fileio_dealloc(fileio *self)
{
if (_PyIOBase_finalize((PyObject *) self) < 0)
return;
_PyObject_GC_UNTRACK(self);
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
if (self->fd >= 0 && self->closefd) {
errno = internal_close(self);
if (errno < 0) {
PySys_WriteStderr("close failed: [Errno %d] %s\n",
errno, strerror(errno));
}
}
Py_CLEAR(self->dict);
Py_TYPE(self)->tp_free((PyObject *)self);
}
@ -351,7 +420,7 @@ err_mode(char *action)
}
static PyObject *
fileio_fileno(PyFileIOObject *self)
fileio_fileno(fileio *self)
{
if (self->fd < 0)
return err_closed();
@ -359,7 +428,7 @@ fileio_fileno(PyFileIOObject *self)
}
static PyObject *
fileio_readable(PyFileIOObject *self)
fileio_readable(fileio *self)
{
if (self->fd < 0)
return err_closed();
@ -367,7 +436,7 @@ fileio_readable(PyFileIOObject *self)
}
static PyObject *
fileio_writable(PyFileIOObject *self)
fileio_writable(fileio *self)
{
if (self->fd < 0)
return err_closed();
@ -375,25 +444,25 @@ fileio_writable(PyFileIOObject *self)
}
static PyObject *
fileio_seekable(PyFileIOObject *self)
fileio_seekable(fileio *self)
{
if (self->fd < 0)
return err_closed();
if (self->seekable < 0) {
int ret;
Py_BEGIN_ALLOW_THREADS
ret = lseek(self->fd, 0, SEEK_CUR);
Py_END_ALLOW_THREADS
if (ret < 0)
PyObject *pos = portable_lseek(self->fd, NULL, SEEK_CUR);
if (pos == NULL) {
PyErr_Clear();
self->seekable = 0;
else
} else {
Py_DECREF(pos);
self->seekable = 1;
}
}
return PyBool_FromLong((long) self->seekable);
}
static PyObject *
fileio_readinto(PyFileIOObject *self, PyObject *args)
fileio_readinto(fileio *self, PyObject *args)
{
Py_buffer pbuf;
Py_ssize_t n;
@ -406,10 +475,13 @@ fileio_readinto(PyFileIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "w*", &pbuf))
return NULL;
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = read(self->fd, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
if (_PyVerify_fd(self->fd)) {
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = read(self->fd, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
} else
n = -1;
PyBuffer_Release(&pbuf);
if (n < 0) {
if (errno == EAGAIN)
@ -421,23 +493,62 @@ fileio_readinto(PyFileIOObject *self, PyObject *args)
return PyLong_FromSsize_t(n);
}
#define DEFAULT_BUFFER_SIZE (8*1024)
static size_t
new_buffersize(fileio *self, size_t currentsize)
{
#ifdef HAVE_FSTAT
off_t pos, end;
struct stat st;
if (fstat(self->fd, &st) == 0) {
end = st.st_size;
pos = lseek(self->fd, 0L, SEEK_CUR);
/* Files claiming a size smaller than SMALLCHUNK may
actually be streaming pseudo-files. In this case, we
apply the more aggressive algorithm below.
*/
if (end >= SMALLCHUNK && end >= pos && pos >= 0) {
/* Add 1 so if the file were to grow we'd notice. */
return currentsize + end - pos + 1;
}
}
#endif
if (currentsize > SMALLCHUNK) {
/* Keep doubling until we reach BIGCHUNK;
then keep adding BIGCHUNK. */
if (currentsize <= BIGCHUNK)
return currentsize + currentsize;
else
return currentsize + BIGCHUNK;
}
return currentsize + SMALLCHUNK;
}
static PyObject *
fileio_readall(PyFileIOObject *self)
fileio_readall(fileio *self)
{
PyObject *result;
Py_ssize_t total = 0;
int n;
result = PyString_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE);
if (!_PyVerify_fd(self->fd))
return PyErr_SetFromErrno(PyExc_IOError);
result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK);
if (result == NULL)
return NULL;
while (1) {
Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE;
if (PyString_GET_SIZE(result) < newsize) {
if (_PyString_Resize(&result, newsize) < 0) {
size_t newsize = new_buffersize(self, total);
if (newsize > PY_SSIZE_T_MAX || newsize <= 0) {
PyErr_SetString(PyExc_OverflowError,
"unbounded read returned more bytes "
"than a Python string can hold ");
Py_DECREF(result);
return NULL;
}
if (PyBytes_GET_SIZE(result) < (Py_ssize_t)newsize) {
if (_PyBytes_Resize(&result, newsize) < 0) {
if (total == 0) {
Py_DECREF(result);
return NULL;
@ -449,7 +560,7 @@ fileio_readall(PyFileIOObject *self)
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = read(self->fd,
PyString_AS_STRING(result) + total,
PyBytes_AS_STRING(result) + total,
newsize - total);
Py_END_ALLOW_THREADS
if (n == 0)
@ -468,8 +579,8 @@ fileio_readall(PyFileIOObject *self)
total += n;
}
if (PyString_GET_SIZE(result) > total) {
if (_PyString_Resize(&result, total) < 0) {
if (PyBytes_GET_SIZE(result) > total) {
if (_PyBytes_Resize(&result, total) < 0) {
/* This should never happen, but just in case */
Py_DECREF(result);
return NULL;
@ -479,7 +590,7 @@ fileio_readall(PyFileIOObject *self)
}
static PyObject *
fileio_read(PyFileIOObject *self, PyObject *args)
fileio_read(fileio *self, PyObject *args)
{
char *ptr;
Py_ssize_t n;
@ -498,17 +609,21 @@ fileio_read(PyFileIOObject *self, PyObject *args)
return fileio_readall(self);
}
bytes = PyString_FromStringAndSize(NULL, size);
bytes = PyBytes_FromStringAndSize(NULL, size);
if (bytes == NULL)
return NULL;
ptr = PyString_AS_STRING(bytes);
ptr = PyBytes_AS_STRING(bytes);
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = read(self->fd, ptr, size);
Py_END_ALLOW_THREADS
if (_PyVerify_fd(self->fd)) {
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = read(self->fd, ptr, size);
Py_END_ALLOW_THREADS
} else
n = -1;
if (n < 0) {
Py_DECREF(bytes);
if (errno == EAGAIN)
Py_RETURN_NONE;
PyErr_SetFromErrno(PyExc_IOError);
@ -516,7 +631,7 @@ fileio_read(PyFileIOObject *self, PyObject *args)
}
if (n != size) {
if (_PyString_Resize(&bytes, n) < 0) {
if (_PyBytes_Resize(&bytes, n) < 0) {
Py_DECREF(bytes);
return NULL;
}
@ -526,7 +641,7 @@ fileio_read(PyFileIOObject *self, PyObject *args)
}
static PyObject *
fileio_write(PyFileIOObject *self, PyObject *args)
fileio_write(fileio *self, PyObject *args)
{
Py_buffer pbuf;
Py_ssize_t n;
@ -539,10 +654,13 @@ fileio_write(PyFileIOObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "s*", &pbuf))
return NULL;
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = write(self->fd, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
if (_PyVerify_fd(self->fd)) {
Py_BEGIN_ALLOW_THREADS
errno = 0;
n = write(self->fd, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
} else
n = -1;
PyBuffer_Release(&pbuf);
@ -558,12 +676,6 @@ fileio_write(PyFileIOObject *self, PyObject *args)
/* XXX Windows support below is likely incomplete */
#if defined(MS_WIN64) || defined(MS_WINDOWS)
typedef PY_LONG_LONG Py_off_t;
#else
typedef off_t Py_off_t;
#endif
/* Cribbed from posix_lseek() */
static PyObject *
portable_lseek(int fd, PyObject *posobj, int whence)
@ -601,13 +713,16 @@ portable_lseek(int fd, PyObject *posobj, int whence)
return NULL;
}
Py_BEGIN_ALLOW_THREADS
if (_PyVerify_fd(fd)) {
Py_BEGIN_ALLOW_THREADS
#if defined(MS_WIN64) || defined(MS_WINDOWS)
res = _lseeki64(fd, pos, whence);
res = _lseeki64(fd, pos, whence);
#else
res = lseek(fd, pos, whence);
res = lseek(fd, pos, whence);
#endif
Py_END_ALLOW_THREADS
Py_END_ALLOW_THREADS
} else
res = -1;
if (res < 0)
return PyErr_SetFromErrno(PyExc_IOError);
@ -619,7 +734,7 @@ portable_lseek(int fd, PyObject *posobj, int whence)
}
static PyObject *
fileio_seek(PyFileIOObject *self, PyObject *args)
fileio_seek(fileio *self, PyObject *args)
{
PyObject *posobj;
int whence = 0;
@ -634,7 +749,7 @@ fileio_seek(PyFileIOObject *self, PyObject *args)
}
static PyObject *
fileio_tell(PyFileIOObject *self, PyObject *args)
fileio_tell(fileio *self, PyObject *args)
{
if (self->fd < 0)
return err_closed();
@ -644,7 +759,7 @@ fileio_tell(PyFileIOObject *self, PyObject *args)
#ifdef HAVE_FTRUNCATE
static PyObject *
fileio_truncate(PyFileIOObject *self, PyObject *args)
fileio_truncate(fileio *self, PyObject *args)
{
PyObject *posobj = NULL;
Py_off_t pos;
@ -670,13 +785,15 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
/* Move to the position to be truncated. */
posobj = portable_lseek(fd, posobj, 0);
}
if (posobj == NULL)
return NULL;
#if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLongLong(posobj);
#else
pos = PyLong_AsLong(posobj);
#endif
if (PyErr_Occurred())
if (pos == -1 && PyErr_Occurred())
return NULL;
#ifdef MS_WINDOWS
@ -714,7 +831,7 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
#endif
static char *
mode_string(PyFileIOObject *self)
mode_string(fileio *self)
{
if (self->readable) {
if (self->writable)
@ -727,17 +844,37 @@ mode_string(PyFileIOObject *self)
}
static PyObject *
fileio_repr(PyFileIOObject *self)
fileio_repr(fileio *self)
{
if (self->fd < 0)
return PyString_FromFormat("_fileio._FileIO(-1)");
PyObject *nameobj, *res;
return PyString_FromFormat("_fileio._FileIO(%d, '%s')",
self->fd, mode_string(self));
if (self->fd < 0)
return PyString_FromFormat("<_io.FileIO [closed]>");
nameobj = PyObject_GetAttrString((PyObject *) self, "name");
if (nameobj == NULL) {
if (PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_Clear();
else
return NULL;
res = PyString_FromFormat("<_io.FileIO fd=%d mode='%s'>",
self->fd, mode_string(self));
}
else {
PyObject *repr = PyObject_Repr(nameobj);
Py_DECREF(nameobj);
if (repr == NULL)
return NULL;
res = PyString_FromFormat("<_io.FileIO name=%s mode='%s'>",
PyString_AS_STRING(repr),
mode_string(self));
Py_DECREF(repr);
}
return res;
}
static PyObject *
fileio_isatty(PyFileIOObject *self)
fileio_isatty(fileio *self)
{
long res;
@ -806,7 +943,7 @@ PyDoc_STRVAR(tell_doc,
"tell() -> int. Current file position");
PyDoc_STRVAR(readinto_doc,
"readinto() -> Undocumented. Don't use this; it may go away.");
"readinto() -> Same as RawIOBase.readinto().");
PyDoc_STRVAR(close_doc,
"close() -> None. Close the file.\n"
@ -848,21 +985,21 @@ static PyMethodDef fileio_methods[] = {
/* 'closed' and 'mode' are attributes for backwards compatibility reasons. */
static PyObject *
get_closed(PyFileIOObject *self, void *closure)
get_closed(fileio *self, void *closure)
{
return PyBool_FromLong((long)(self->fd < 0));
}
static PyObject *
get_closefd(PyFileIOObject *self, void *closure)
get_closefd(fileio *self, void *closure)
{
return PyBool_FromLong((long)(self->closefd));
}
static PyObject *
get_mode(PyFileIOObject *self, void *closure)
get_mode(fileio *self, void *closure)
{
return PyString_FromString(mode_string(self));
return PyUnicode_FromString(mode_string(self));
}
static PyGetSetDef fileio_getsetlist[] = {
@ -870,19 +1007,19 @@ static PyGetSetDef fileio_getsetlist[] = {
{"closefd", (getter)get_closefd, NULL,
"True if the file descriptor will be closed"},
{"mode", (getter)get_mode, NULL, "String giving the file mode"},
{0},
{NULL},
};
PyTypeObject PyFileIO_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_FileIO",
sizeof(PyFileIOObject),
"_io.FileIO",
sizeof(fileio),
0,
(destructor)fileio_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_reserved */
(reprfunc)fileio_repr, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
@ -893,12 +1030,13 @@ PyTypeObject PyFileIO_Type = {
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
| Py_TPFLAGS_HAVE_GC, /* tp_flags */
fileio_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
(traverseproc)fileio_traverse, /* tp_traverse */
(inquiry)fileio_clear, /* tp_clear */
0, /* tp_richcompare */
offsetof(PyFileIOObject, weakreflist), /* tp_weaklistoffset */
offsetof(fileio, weakreflist), /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
fileio_methods, /* tp_methods */
@ -908,28 +1046,9 @@ PyTypeObject PyFileIO_Type = {
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
offsetof(fileio, dict), /* tp_dictoffset */
fileio_init, /* tp_init */
PyType_GenericAlloc, /* tp_alloc */
fileio_new, /* tp_new */
PyObject_Del, /* tp_free */
PyObject_GC_Del, /* tp_free */
};
static PyMethodDef module_methods[] = {
{NULL, NULL}
};
PyMODINIT_FUNC
init_fileio(void)
{
PyObject *m; /* a module object */
m = Py_InitModule3("_fileio", module_methods,
"Fast implementation of io.FileIO.");
if (m == NULL)
return;
if (PyType_Ready(&PyFileIO_Type) < 0)
return;
Py_INCREF(&PyFileIO_Type);
PyModule_AddObject(m, "_FileIO", (PyObject *) &PyFileIO_Type);
}

894
Modules/_io/iobase.c Normal file
View File

@ -0,0 +1,894 @@
/*
An implementation of the I/O abstract base classes hierarchy
as defined by PEP 3116 - "New I/O"
Classes defined here: IOBase, RawIOBase.
Written by Amaury Forgeot d'Arc and Antoine Pitrou
*/
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "structmember.h"
#include "_iomodule.h"
/*
* IOBase class, an abstract class
*/
typedef struct {
PyObject_HEAD
PyObject *dict;
PyObject *weakreflist;
} iobase;
PyDoc_STRVAR(iobase_doc,
"The abstract base class for all I/O classes, acting on streams of\n"
"bytes. There is no public constructor.\n"
"\n"
"This class provides dummy implementations for many methods that\n"
"derived classes can override selectively; the default implementations\n"
"represent a file that cannot be read, written or seeked.\n"
"\n"
"Even though IOBase does not declare read, readinto, or write because\n"
"their signatures will vary, implementations and clients should\n"
"consider those methods part of the interface. Also, implementations\n"
"may raise a IOError when operations they do not support are called.\n"
"\n"
"The basic type used for binary data read from or written to a file is\n"
"bytes. bytearrays are accepted too, and in some cases (such as\n"
"readinto) needed. Text I/O classes work with str data.\n"
"\n"
"Note that calling any method (even inquiries) on a closed stream is\n"
"undefined. Implementations may raise IOError in this case.\n"
"\n"
"IOBase (and its subclasses) support the iterator protocol, meaning\n"
"that an IOBase object can be iterated over yielding the lines in a\n"
"stream.\n"
"\n"
"IOBase also supports the :keyword:`with` statement. In this example,\n"
"fp is closed after the suite of the with statment is complete:\n"
"\n"
"with open('spam.txt', 'r') as fp:\n"
" fp.write('Spam and eggs!')\n");
/* Use this macro whenever you want to check the internal `closed` status
of the IOBase object rather than the virtual `closed` attribute as returned
by whatever subclass. */
#define IS_CLOSED(self) \
PyObject_HasAttrString(self, "__IOBase_closed")
/* Internal methods */
static PyObject *
iobase_unsupported(const char *message)
{
PyErr_SetString(_PyIO_unsupported_operation, message);
return NULL;
}
/* Positionning */
PyDoc_STRVAR(iobase_seek_doc,
"Change stream position.\n"
"\n"
"Change the stream position to byte offset offset. offset is\n"
"interpreted relative to the position indicated by whence. Values\n"
"for whence are:\n"
"\n"
"* 0 -- start of stream (the default); offset should be zero or positive\n"
"* 1 -- current stream position; offset may be negative\n"
"* 2 -- end of stream; offset is usually negative\n"
"\n"
"Return the new absolute position.");
static PyObject *
iobase_seek(PyObject *self, PyObject *args)
{
return iobase_unsupported("seek");
}
PyDoc_STRVAR(iobase_tell_doc,
"Return current stream position.");
static PyObject *
iobase_tell(PyObject *self, PyObject *args)
{
return PyObject_CallMethod(self, "seek", "ii", 0, 1);
}
PyDoc_STRVAR(iobase_truncate_doc,
"Truncate file to size bytes.\n"
"\n"
"Size defaults to the current IO position as reported by tell(). Return\n"
"the new size.");
static PyObject *
iobase_truncate(PyObject *self, PyObject *args)
{
return iobase_unsupported("truncate");
}
/* Flush and close methods */
PyDoc_STRVAR(iobase_flush_doc,
"Flush write buffers, if applicable.\n"
"\n"
"This is not implemented for read-only and non-blocking streams.\n");
static PyObject *
iobase_flush(PyObject *self, PyObject *args)
{
/* XXX Should this return the number of bytes written??? */
if (IS_CLOSED(self)) {
PyErr_SetString(PyExc_ValueError, "I/O operation on closed file.");
return NULL;
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(iobase_close_doc,
"Flush and close the IO object.\n"
"\n"
"This method has no effect if the file is already closed.\n");
static int
iobase_closed(PyObject *self)
{
PyObject *res;
int closed;
/* This gets the derived attribute, which is *not* __IOBase_closed
in most cases! */
res = PyObject_GetAttr(self, _PyIO_str_closed);
if (res == NULL)
return 0;
closed = PyObject_IsTrue(res);
Py_DECREF(res);
return closed;
}
static PyObject *
iobase_closed_get(PyObject *self, void *context)
{
return PyBool_FromLong(IS_CLOSED(self));
}
PyObject *
_PyIOBase_check_closed(PyObject *self, PyObject *args)
{
if (iobase_closed(self)) {
PyErr_SetString(PyExc_ValueError, "I/O operation on closed file.");
return NULL;
}
if (args == Py_True)
return Py_None;
else
Py_RETURN_NONE;
}
/* XXX: IOBase thinks it has to maintain its own internal state in
`__IOBase_closed` and call flush() by itself, but it is redundant with
whatever behaviour a non-trivial derived class will implement. */
static PyObject *
iobase_close(PyObject *self, PyObject *args)
{
PyObject *res;
if (IS_CLOSED(self))
Py_RETURN_NONE;
res = PyObject_CallMethodObjArgs(self, _PyIO_str_flush, NULL);
PyObject_SetAttrString(self, "__IOBase_closed", Py_True);
if (res == NULL) {
/* If flush() fails, just give up */
if (PyErr_ExceptionMatches(PyExc_IOError))
PyErr_Clear();
else
return NULL;
}
Py_XDECREF(res);
Py_RETURN_NONE;
}
/* Finalization and garbage collection support */
int
_PyIOBase_finalize(PyObject *self)
{
PyObject *res;
PyObject *tp, *v, *tb;
int closed = 1;
int is_zombie;
/* If _PyIOBase_finalize() is called from a destructor, we need to
resurrect the object as calling close() can invoke arbitrary code. */
is_zombie = (Py_REFCNT(self) == 0);
if (is_zombie) {
++Py_REFCNT(self);
}
PyErr_Fetch(&tp, &v, &tb);
/* If `closed` doesn't exist or can't be evaluated as bool, then the
object is probably in an unusable state, so ignore. */
res = PyObject_GetAttr(self, _PyIO_str_closed);
if (res == NULL)
PyErr_Clear();
else {
closed = PyObject_IsTrue(res);
Py_DECREF(res);
if (closed == -1)
PyErr_Clear();
}
if (closed == 0) {
res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_close,
NULL);
/* Silencing I/O errors is bad, but printing spurious tracebacks is
equally as bad, and potentially more frequent (because of
shutdown issues). */
if (res == NULL)
PyErr_Clear();
else
Py_DECREF(res);
}
PyErr_Restore(tp, v, tb);
if (is_zombie) {
if (--Py_REFCNT(self) != 0) {
/* The object lives again. The following code is taken from
slot_tp_del in typeobject.c. */
Py_ssize_t refcnt = Py_REFCNT(self);
_Py_NewReference(self);
Py_REFCNT(self) = refcnt;
/* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
* we need to undo that. */
_Py_DEC_REFTOTAL;
/* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
* chain, so no more to do there.
* If COUNT_ALLOCS, the original decref bumped tp_frees, and
* _Py_NewReference bumped tp_allocs: both of those need to be
* undone.
*/
#ifdef COUNT_ALLOCS
--Py_TYPE(self)->tp_frees;
--Py_TYPE(self)->tp_allocs;
#endif
return -1;
}
}
return 0;
}
static int
iobase_traverse(iobase *self, visitproc visit, void *arg)
{
Py_VISIT(self->dict);
return 0;
}
static int
iobase_clear(iobase *self)
{
if (_PyIOBase_finalize((PyObject *) self) < 0)
return -1;
Py_CLEAR(self->dict);
return 0;
}
/* Destructor */
static void
iobase_dealloc(iobase *self)
{
/* NOTE: since IOBaseObject has its own dict, Python-defined attributes
are still available here for close() to use.
However, if the derived class declares a __slots__, those slots are
already gone.
*/
if (_PyIOBase_finalize((PyObject *) self) < 0) {
/* When called from a heap type's dealloc, the type will be
decref'ed on return (see e.g. subtype_dealloc in typeobject.c). */
if (PyType_HasFeature(Py_TYPE(self), Py_TPFLAGS_HEAPTYPE))
Py_INCREF(Py_TYPE(self));
return;
}
_PyObject_GC_UNTRACK(self);
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
Py_CLEAR(self->dict);
Py_TYPE(self)->tp_free((PyObject *) self);
}
/* Inquiry methods */
PyDoc_STRVAR(iobase_seekable_doc,
"Return whether object supports random access.\n"
"\n"
"If False, seek(), tell() and truncate() will raise IOError.\n"
"This method may need to do a test seek().");
static PyObject *
iobase_seekable(PyObject *self, PyObject *args)
{
Py_RETURN_FALSE;
}
PyObject *
_PyIOBase_check_seekable(PyObject *self, PyObject *args)
{
PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_seekable, NULL);
if (res == NULL)
return NULL;
if (res != Py_True) {
Py_CLEAR(res);
PyErr_SetString(PyExc_IOError, "File or stream is not seekable.");
return NULL;
}
if (args == Py_True) {
Py_DECREF(res);
}
return res;
}
PyDoc_STRVAR(iobase_readable_doc,
"Return whether object was opened for reading.\n"
"\n"
"If False, read() will raise IOError.");
static PyObject *
iobase_readable(PyObject *self, PyObject *args)
{
Py_RETURN_FALSE;
}
/* May be called with any object */
PyObject *
_PyIOBase_check_readable(PyObject *self, PyObject *args)
{
PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_readable, NULL);
if (res == NULL)
return NULL;
if (res != Py_True) {
Py_CLEAR(res);
PyErr_SetString(PyExc_IOError, "File or stream is not readable.");
return NULL;
}
if (args == Py_True) {
Py_DECREF(res);
}
return res;
}
PyDoc_STRVAR(iobase_writable_doc,
"Return whether object was opened for writing.\n"
"\n"
"If False, read() will raise IOError.");
static PyObject *
iobase_writable(PyObject *self, PyObject *args)
{
Py_RETURN_FALSE;
}
/* May be called with any object */
PyObject *
_PyIOBase_check_writable(PyObject *self, PyObject *args)
{
PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_writable, NULL);
if (res == NULL)
return NULL;
if (res != Py_True) {
Py_CLEAR(res);
PyErr_SetString(PyExc_IOError, "File or stream is not writable.");
return NULL;
}
if (args == Py_True) {
Py_DECREF(res);
}
return res;
}
/* Context manager */
static PyObject *
iobase_enter(PyObject *self, PyObject *args)
{
if (_PyIOBase_check_closed(self, Py_True) == NULL)
return NULL;
Py_INCREF(self);
return self;
}
static PyObject *
iobase_exit(PyObject *self, PyObject *args)
{
return PyObject_CallMethodObjArgs(self, _PyIO_str_close, NULL);
}
/* Lower-level APIs */
/* XXX Should these be present even if unimplemented? */
PyDoc_STRVAR(iobase_fileno_doc,
"Returns underlying file descriptor if one exists.\n"
"\n"
"An IOError is raised if the IO object does not use a file descriptor.\n");
static PyObject *
iobase_fileno(PyObject *self, PyObject *args)
{
return iobase_unsupported("fileno");
}
PyDoc_STRVAR(iobase_isatty_doc,
"Return whether this is an 'interactive' stream.\n"
"\n"
"Return False if it can't be determined.\n");
static PyObject *
iobase_isatty(PyObject *self, PyObject *args)
{
if (_PyIOBase_check_closed(self, Py_True) == NULL)
return NULL;
Py_RETURN_FALSE;
}
/* Readline(s) and writelines */
PyDoc_STRVAR(iobase_readline_doc,
"Read and return a line from the stream.\n"
"\n"
"If limit is specified, at most limit bytes will be read.\n"
"\n"
"The line terminator is always b'\n' for binary files; for text\n"
"files, the newlines argument to open can be used to select the line\n"
"terminator(s) recognized.\n");
static PyObject *
iobase_readline(PyObject *self, PyObject *args)
{
/* For backwards compatibility, a (slowish) readline(). */
Py_ssize_t limit = -1;
int has_peek = 0;
PyObject *buffer, *result;
Py_ssize_t old_size = -1;
if (!PyArg_ParseTuple(args, "|n:readline", &limit)) {
return NULL;
}
if (PyObject_HasAttrString(self, "peek"))
has_peek = 1;
buffer = PyByteArray_FromStringAndSize(NULL, 0);
if (buffer == NULL)
return NULL;
while (limit < 0 || Py_SIZE(buffer) < limit) {
Py_ssize_t nreadahead = 1;
PyObject *b;
if (has_peek) {
PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1);
if (readahead == NULL)
goto fail;
if (!PyBytes_Check(readahead)) {
PyErr_Format(PyExc_IOError,
"peek() should have returned a bytes object, "
"not '%.200s'", Py_TYPE(readahead)->tp_name);
Py_DECREF(readahead);
goto fail;
}
if (PyBytes_GET_SIZE(readahead) > 0) {
Py_ssize_t n = 0;
const char *buf = PyBytes_AS_STRING(readahead);
if (limit >= 0) {
do {
if (n >= PyBytes_GET_SIZE(readahead) || n >= limit)
break;
if (buf[n++] == '\n')
break;
} while (1);
}
else {
do {
if (n >= PyBytes_GET_SIZE(readahead))
break;
if (buf[n++] == '\n')
break;
} while (1);
}
nreadahead = n;
}
Py_DECREF(readahead);
}
b = PyObject_CallMethod(self, "read", "n", nreadahead);
if (b == NULL)
goto fail;
if (!PyBytes_Check(b)) {
PyErr_Format(PyExc_IOError,
"read() should have returned a bytes object, "
"not '%.200s'", Py_TYPE(b)->tp_name);
Py_DECREF(b);
goto fail;
}
if (PyBytes_GET_SIZE(b) == 0) {
Py_DECREF(b);
break;
}
old_size = PyByteArray_GET_SIZE(buffer);
PyByteArray_Resize(buffer, old_size + PyBytes_GET_SIZE(b));
memcpy(PyByteArray_AS_STRING(buffer) + old_size,
PyBytes_AS_STRING(b), PyBytes_GET_SIZE(b));
Py_DECREF(b);
if (PyByteArray_AS_STRING(buffer)[PyByteArray_GET_SIZE(buffer) - 1] == '\n')
break;
}
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(buffer),
PyByteArray_GET_SIZE(buffer));
Py_DECREF(buffer);
return result;
fail:
Py_DECREF(buffer);
return NULL;
}
static PyObject *
iobase_iter(PyObject *self)
{
if (_PyIOBase_check_closed(self, Py_True) == NULL)
return NULL;
Py_INCREF(self);
return self;
}
static PyObject *
iobase_iternext(PyObject *self)
{
PyObject *line = PyObject_CallMethodObjArgs(self, _PyIO_str_readline, NULL);
if (line == NULL)
return NULL;
if (PyObject_Size(line) == 0) {
Py_DECREF(line);
return NULL;
}
return line;
}
PyDoc_STRVAR(iobase_readlines_doc,
"Return a list of lines from the stream.\n"
"\n"
"hint can be specified to control the number of lines read: no more\n"
"lines will be read if the total size (in bytes/characters) of all\n"
"lines so far exceeds hint.");
static PyObject *
iobase_readlines(PyObject *self, PyObject *args)
{
Py_ssize_t hint = -1, length = 0;
PyObject *hintobj = Py_None, *result;
if (!PyArg_ParseTuple(args, "|O:readlines", &hintobj)) {
return NULL;
}
if (hintobj != Py_None) {
hint = PyNumber_AsSsize_t(hintobj, PyExc_ValueError);
if (hint == -1 && PyErr_Occurred())
return NULL;
}
result = PyList_New(0);
if (result == NULL)
return NULL;
if (hint <= 0) {
/* XXX special-casing this made sense in the Python version in order
to remove the bytecode interpretation overhead, but it could
probably be removed here. */
PyObject *ret = PyObject_CallMethod(result, "extend", "O", self);
if (ret == NULL) {
Py_DECREF(result);
return NULL;
}
Py_DECREF(ret);
return result;
}
while (1) {
PyObject *line = PyIter_Next(self);
if (line == NULL) {
if (PyErr_Occurred()) {
Py_DECREF(result);
return NULL;
}
else
break; /* StopIteration raised */
}
if (PyList_Append(result, line) < 0) {
Py_DECREF(line);
Py_DECREF(result);
return NULL;
}
length += PyObject_Size(line);
Py_DECREF(line);
if (length > hint)
break;
}
return result;
}
static PyObject *
iobase_writelines(PyObject *self, PyObject *args)
{
PyObject *lines, *iter, *res;
if (!PyArg_ParseTuple(args, "O:writelines", &lines)) {
return NULL;
}
if (_PyIOBase_check_closed(self, Py_True) == NULL)
return NULL;
iter = PyObject_GetIter(lines);
if (iter == NULL)
return NULL;
while (1) {
PyObject *line = PyIter_Next(iter);
if (line == NULL) {
if (PyErr_Occurred()) {
Py_DECREF(iter);
return NULL;
}
else
break; /* Stop Iteration */
}
res = PyObject_CallMethodObjArgs(self, _PyIO_str_write, line, NULL);
Py_DECREF(line);
if (res == NULL) {
Py_DECREF(iter);
return NULL;
}
Py_DECREF(res);
}
Py_DECREF(iter);
Py_RETURN_NONE;
}
static PyMethodDef iobase_methods[] = {
{"seek", iobase_seek, METH_VARARGS, iobase_seek_doc},
{"tell", iobase_tell, METH_NOARGS, iobase_tell_doc},
{"truncate", iobase_truncate, METH_VARARGS, iobase_truncate_doc},
{"flush", iobase_flush, METH_NOARGS, iobase_flush_doc},
{"close", iobase_close, METH_NOARGS, iobase_close_doc},
{"seekable", iobase_seekable, METH_NOARGS, iobase_seekable_doc},
{"readable", iobase_readable, METH_NOARGS, iobase_readable_doc},
{"writable", iobase_writable, METH_NOARGS, iobase_writable_doc},
{"_checkClosed", _PyIOBase_check_closed, METH_NOARGS},
{"_checkSeekable", _PyIOBase_check_seekable, METH_NOARGS},
{"_checkReadable", _PyIOBase_check_readable, METH_NOARGS},
{"_checkWritable", _PyIOBase_check_writable, METH_NOARGS},
{"fileno", iobase_fileno, METH_NOARGS, iobase_fileno_doc},
{"isatty", iobase_isatty, METH_NOARGS, iobase_isatty_doc},
{"__enter__", iobase_enter, METH_NOARGS},
{"__exit__", iobase_exit, METH_VARARGS},
{"readline", iobase_readline, METH_VARARGS, iobase_readline_doc},
{"readlines", iobase_readlines, METH_VARARGS, iobase_readlines_doc},
{"writelines", iobase_writelines, METH_VARARGS},
{NULL, NULL}
};
static PyGetSetDef iobase_getset[] = {
{"closed", (getter)iobase_closed_get, NULL, NULL},
{NULL}
};
PyTypeObject PyIOBase_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_io._IOBase", /*tp_name*/
sizeof(iobase), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)iobase_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare */
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
iobase_doc, /* tp_doc */
(traverseproc)iobase_traverse, /* tp_traverse */
(inquiry)iobase_clear, /* tp_clear */
0, /* tp_richcompare */
offsetof(iobase, weakreflist), /* tp_weaklistoffset */
iobase_iter, /* tp_iter */
iobase_iternext, /* tp_iternext */
iobase_methods, /* tp_methods */
0, /* tp_members */
iobase_getset, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
offsetof(iobase, dict), /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
PyType_GenericNew, /* tp_new */
};
/*
* RawIOBase class, Inherits from IOBase.
*/
PyDoc_STRVAR(rawiobase_doc,
"Base class for raw binary I/O.");
/*
* The read() method is implemented by calling readinto(); derived classes
* that want to support read() only need to implement readinto() as a
* primitive operation. In general, readinto() can be more efficient than
* read().
*
* (It would be tempting to also provide an implementation of readinto() in
* terms of read(), in case the latter is a more suitable primitive operation,
* but that would lead to nasty recursion in case a subclass doesn't implement
* either.)
*/
static PyObject *
rawiobase_read(PyObject *self, PyObject *args)
{
Py_ssize_t n = -1;
PyObject *b, *res;
if (!PyArg_ParseTuple(args, "|n:read", &n)) {
return NULL;
}
if (n < 0)
return PyObject_CallMethod(self, "readall", NULL);
/* TODO: allocate a bytes object directly instead and manually construct
a writable memoryview pointing to it. */
b = PyByteArray_FromStringAndSize(NULL, n);
if (b == NULL)
return NULL;
res = PyObject_CallMethodObjArgs(self, _PyIO_str_readinto, b, NULL);
if (res == NULL) {
Py_DECREF(b);
return NULL;
}
n = PyNumber_AsSsize_t(res, PyExc_ValueError);
Py_DECREF(res);
if (n == -1 && PyErr_Occurred()) {
Py_DECREF(b);
return NULL;
}
res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n);
Py_DECREF(b);
return res;
}
PyDoc_STRVAR(rawiobase_readall_doc,
"Read until EOF, using multiple read() call.");
static PyObject *
rawiobase_readall(PyObject *self, PyObject *args)
{
int r;
PyObject *chunks = PyList_New(0);
PyObject *result;
if (chunks == NULL)
return NULL;
while (1) {
PyObject *data = PyObject_CallMethod(self, "read",
"i", DEFAULT_BUFFER_SIZE);
if (!data) {
Py_DECREF(chunks);
return NULL;
}
if (!PyBytes_Check(data)) {
Py_DECREF(chunks);
Py_DECREF(data);
PyErr_SetString(PyExc_TypeError, "read() should return bytes");
return NULL;
}
if (PyBytes_GET_SIZE(data) == 0) {
/* EOF */
Py_DECREF(data);
break;
}
r = PyList_Append(chunks, data);
Py_DECREF(data);
if (r < 0) {
Py_DECREF(chunks);
return NULL;
}
}
result = _PyBytes_Join(_PyIO_empty_bytes, chunks);
Py_DECREF(chunks);
return result;
}
static PyMethodDef rawiobase_methods[] = {
{"read", rawiobase_read, METH_VARARGS},
{"readall", rawiobase_readall, METH_NOARGS, rawiobase_readall_doc},
{NULL, NULL}
};
PyTypeObject PyRawIOBase_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_io._RawIOBase", /*tp_name*/
0, /*tp_basicsize*/
0, /*tp_itemsize*/
0, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare */
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
rawiobase_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
rawiobase_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
&PyIOBase_Type, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
};

756
Modules/_io/stringio.c Normal file
View File

@ -0,0 +1,756 @@
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "structmember.h"
#include "_iomodule.h"
/* Implementation note: the buffer is always at least one character longer
than the enclosed string, for proper functioning of _PyIO_find_line_ending.
*/
typedef struct {
PyObject_HEAD
Py_UNICODE *buf;
Py_ssize_t pos;
Py_ssize_t string_size;
size_t buf_size;
char ok; /* initialized? */
char closed;
char readuniversal;
char readtranslate;
PyObject *decoder;
PyObject *readnl;
PyObject *writenl;
PyObject *dict;
PyObject *weakreflist;
} stringio;
#define CHECK_INITIALIZED(self) \
if (self->ok <= 0) { \
PyErr_SetString(PyExc_ValueError, \
"I/O operation on uninitialized object"); \
return NULL; \
}
#define CHECK_CLOSED(self) \
if (self->closed) { \
PyErr_SetString(PyExc_ValueError, \
"I/O operation on closed file"); \
return NULL; \
}
PyDoc_STRVAR(stringio_doc,
"Text I/O implementation using an in-memory buffer.\n"
"\n"
"The initial_value argument sets the value of object. The newline\n"
"argument is like the one of TextIOWrapper's constructor.");
/* Internal routine for changing the size, in terms of characters, of the
buffer of StringIO objects. The caller should ensure that the 'size'
argument is non-negative. Returns 0 on success, -1 otherwise. */
static int
resize_buffer(stringio *self, size_t size)
{
/* Here, unsigned types are used to avoid dealing with signed integer
overflow, which is undefined in C. */
size_t alloc = self->buf_size;
Py_UNICODE *new_buf = NULL;
assert(self->buf != NULL);
/* Reserve one more char for line ending detection. */
size = size + 1;
/* For simplicity, stay in the range of the signed type. Anyway, Python
doesn't allow strings to be longer than this. */
if (size > PY_SSIZE_T_MAX)
goto overflow;
if (size < alloc / 2) {
/* Major downsize; resize down to exact size. */
alloc = size + 1;
}
else if (size < alloc) {
/* Within allocated size; quick exit */
return 0;
}
else if (size <= alloc * 1.125) {
/* Moderate upsize; overallocate similar to list_resize() */
alloc = size + (size >> 3) + (size < 9 ? 3 : 6);
}
else {
/* Major upsize; resize up to exact size */
alloc = size + 1;
}
if (alloc > ((size_t)-1) / sizeof(Py_UNICODE))
goto overflow;
new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf,
alloc * sizeof(Py_UNICODE));
if (new_buf == NULL) {
PyErr_NoMemory();
return -1;
}
self->buf_size = alloc;
self->buf = new_buf;
return 0;
overflow:
PyErr_SetString(PyExc_OverflowError,
"new buffer size too large");
return -1;
}
/* Internal routine for writing a whole PyUnicode object to the buffer of a
StringIO object. Returns 0 on success, or -1 on error. */
static Py_ssize_t
write_str(stringio *self, PyObject *obj)
{
Py_UNICODE *str;
Py_ssize_t len;
PyObject *decoded = NULL;
assert(self->buf != NULL);
assert(self->pos >= 0);
if (self->decoder != NULL) {
decoded = _PyIncrementalNewlineDecoder_decode(
self->decoder, obj, 1 /* always final */);
}
else {
decoded = obj;
Py_INCREF(decoded);
}
if (self->writenl) {
PyObject *translated = PyUnicode_Replace(
decoded, _PyIO_str_nl, self->writenl, -1);
Py_DECREF(decoded);
decoded = translated;
}
if (decoded == NULL)
return -1;
assert(PyUnicode_Check(decoded));
str = PyUnicode_AS_UNICODE(decoded);
len = PyUnicode_GET_SIZE(decoded);
assert(len >= 0);
/* This overflow check is not strictly necessary. However, it avoids us to
deal with funky things like comparing an unsigned and a signed
integer. */
if (self->pos > PY_SSIZE_T_MAX - len) {
PyErr_SetString(PyExc_OverflowError,
"new position too large");
goto fail;
}
if (self->pos + len > self->string_size) {
if (resize_buffer(self, self->pos + len) < 0)
goto fail;
}
if (self->pos > self->string_size) {
/* In case of overseek, pad with null bytes the buffer region between
the end of stream and the current position.
0 lo string_size hi
| |<---used--->|<----------available----------->|
| | <--to pad-->|<---to write---> |
0 buf positon
*/
memset(self->buf + self->string_size, '\0',
(self->pos - self->string_size) * sizeof(Py_UNICODE));
}
/* Copy the data to the internal buffer, overwriting some of the
existing data if self->pos < self->string_size. */
memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE));
self->pos += len;
/* Set the new length of the internal string if it has changed. */
if (self->string_size < self->pos) {
self->string_size = self->pos;
}
Py_DECREF(decoded);
return 0;
fail:
Py_XDECREF(decoded);
return -1;
}
PyDoc_STRVAR(stringio_getvalue_doc,
"Retrieve the entire contents of the object.");
static PyObject *
stringio_getvalue(stringio *self)
{
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
return PyUnicode_FromUnicode(self->buf, self->string_size);
}
PyDoc_STRVAR(stringio_tell_doc,
"Tell the current file position.");
static PyObject *
stringio_tell(stringio *self)
{
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
return PyLong_FromSsize_t(self->pos);
}
PyDoc_STRVAR(stringio_read_doc,
"Read at most n characters, returned as a string.\n"
"\n"
"If the argument is negative or omitted, read until EOF\n"
"is reached. Return an empty string at EOF.\n");
static PyObject *
stringio_read(stringio *self, PyObject *args)
{
Py_ssize_t size, n;
Py_UNICODE *output;
PyObject *arg = Py_None;
CHECK_INITIALIZED(self);
if (!PyArg_ParseTuple(args, "|O:read", &arg))
return NULL;
CHECK_CLOSED(self);
if (PyNumber_Check(arg)) {
size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred())
return NULL;
}
else if (arg == Py_None) {
/* Read until EOF is reached, by default. */
size = -1;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
/* adjust invalid sizes */
n = self->string_size - self->pos;
if (size < 0 || size > n) {
size = n;
if (size < 0)
size = 0;
}
output = self->buf + self->pos;
self->pos += size;
return PyUnicode_FromUnicode(output, size);
}
/* Internal helper, used by stringio_readline and stringio_iternext */
static PyObject *
_stringio_readline(stringio *self, Py_ssize_t limit)
{
Py_UNICODE *start, *end, old_char;
Py_ssize_t len, consumed;
/* In case of overseek, return the empty string */
if (self->pos >= self->string_size)
return PyUnicode_FromString("");
start = self->buf + self->pos;
if (limit < 0 || limit > self->string_size - self->pos)
limit = self->string_size - self->pos;
end = start + limit;
old_char = *end;
*end = '\0';
len = _PyIO_find_line_ending(
self->readtranslate, self->readuniversal, self->readnl,
start, end, &consumed);
*end = old_char;
/* If we haven't found any line ending, we just return everything
(`consumed` is ignored). */
if (len < 0)
len = limit;
self->pos += len;
return PyUnicode_FromUnicode(start, len);
}
PyDoc_STRVAR(stringio_readline_doc,
"Read until newline or EOF.\n"
"\n"
"Returns an empty string if EOF is hit immediately.\n");
static PyObject *
stringio_readline(stringio *self, PyObject *args)
{
PyObject *arg = Py_None;
Py_ssize_t limit = -1;
CHECK_INITIALIZED(self);
if (!PyArg_ParseTuple(args, "|O:readline", &arg))
return NULL;
CHECK_CLOSED(self);
if (PyNumber_Check(arg)) {
limit = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (limit == -1 && PyErr_Occurred())
return NULL;
}
else if (arg != Py_None) {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
return _stringio_readline(self, limit);
}
static PyObject *
stringio_iternext(stringio *self)
{
PyObject *line;
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
if (Py_TYPE(self) == &PyStringIO_Type) {
/* Skip method call overhead for speed */
line = _stringio_readline(self, -1);
}
else {
/* XXX is subclassing StringIO really supported? */
line = PyObject_CallMethodObjArgs((PyObject *)self,
_PyIO_str_readline, NULL);
if (line && !PyUnicode_Check(line)) {
PyErr_Format(PyExc_IOError,
"readline() should have returned an str object, "
"not '%.200s'", Py_TYPE(line)->tp_name);
Py_DECREF(line);
return NULL;
}
}
if (line == NULL)
return NULL;
if (PyUnicode_GET_SIZE(line) == 0) {
/* Reached EOF */
Py_DECREF(line);
return NULL;
}
return line;
}
PyDoc_STRVAR(stringio_truncate_doc,
"Truncate size to pos.\n"
"\n"
"The pos argument defaults to the current file position, as\n"
"returned by tell(). Imply an absolute seek to pos.\n"
"Returns the new absolute position.\n");
static PyObject *
stringio_truncate(stringio *self, PyObject *args)
{
Py_ssize_t size;
PyObject *arg = Py_None;
CHECK_INITIALIZED(self);
if (!PyArg_ParseTuple(args, "|O:truncate", &arg))
return NULL;
CHECK_CLOSED(self);
if (PyNumber_Check(arg)) {
size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
if (size == -1 && PyErr_Occurred())
return NULL;
}
else if (arg == Py_None) {
/* Truncate to current position if no argument is passed. */
size = self->pos;
}
else {
PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
Py_TYPE(arg)->tp_name);
return NULL;
}
if (size < 0) {
PyErr_Format(PyExc_ValueError,
"Negative size value %zd", size);
return NULL;
}
if (size < self->string_size) {
if (resize_buffer(self, size) < 0)
return NULL;
self->string_size = size;
}
self->pos = size;
return PyLong_FromSsize_t(size);
}
PyDoc_STRVAR(stringio_seek_doc,
"Change stream position.\n"
"\n"
"Seek to character offset pos relative to position indicated by whence:\n"
" 0 Start of stream (the default). pos should be >= 0;\n"
" 1 Current position - pos must be 0;\n"
" 2 End of stream - pos must be 0.\n"
"Returns the new absolute position.\n");
static PyObject *
stringio_seek(stringio *self, PyObject *args)
{
PyObject *posobj;
Py_ssize_t pos;
int mode = 0;
CHECK_INITIALIZED(self);
if (!PyArg_ParseTuple(args, "O|i:seek", &posobj, &mode))
return NULL;
pos = PyNumber_AsSsize_t(posobj, PyExc_OverflowError);
if (pos == -1 && PyErr_Occurred())
return NULL;
CHECK_CLOSED(self);
if (mode != 0 && mode != 1 && mode != 2) {
PyErr_Format(PyExc_ValueError,
"Invalid whence (%i, should be 0, 1 or 2)", mode);
return NULL;
}
else if (pos < 0 && mode == 0) {
PyErr_Format(PyExc_ValueError,
"Negative seek position %zd", pos);
return NULL;
}
else if (mode != 0 && pos != 0) {
PyErr_SetString(PyExc_IOError,
"Can't do nonzero cur-relative seeks");
return NULL;
}
/* mode 0: offset relative to beginning of the string.
mode 1: no change to current position.
mode 2: change position to end of file. */
if (mode == 1) {
pos = self->pos;
}
else if (mode == 2) {
pos = self->string_size;
}
self->pos = pos;
return PyLong_FromSsize_t(self->pos);
}
PyDoc_STRVAR(stringio_write_doc,
"Write string to file.\n"
"\n"
"Returns the number of characters written, which is always equal to\n"
"the length of the string.\n");
static PyObject *
stringio_write(stringio *self, PyObject *obj)
{
Py_ssize_t size;
CHECK_INITIALIZED(self);
if (!PyUnicode_Check(obj)) {
PyErr_Format(PyExc_TypeError, "string argument expected, got '%s'",
Py_TYPE(obj)->tp_name);
return NULL;
}
CHECK_CLOSED(self);
size = PyUnicode_GET_SIZE(obj);
if (size > 0 && write_str(self, obj) < 0)
return NULL;
return PyLong_FromSsize_t(size);
}
PyDoc_STRVAR(stringio_close_doc,
"Close the IO object. Attempting any further operation after the\n"
"object is closed will raise a ValueError.\n"
"\n"
"This method has no effect if the file is already closed.\n");
static PyObject *
stringio_close(stringio *self)
{
self->closed = 1;
/* Free up some memory */
if (resize_buffer(self, 0) < 0)
return NULL;
Py_CLEAR(self->readnl);
Py_CLEAR(self->writenl);
Py_CLEAR(self->decoder);
Py_RETURN_NONE;
}
static int
stringio_traverse(stringio *self, visitproc visit, void *arg)
{
Py_VISIT(self->dict);
return 0;
}
static int
stringio_clear(stringio *self)
{
Py_CLEAR(self->dict);
return 0;
}
static void
stringio_dealloc(stringio *self)
{
_PyObject_GC_UNTRACK(self);
Py_CLEAR(self->readnl);
Py_CLEAR(self->writenl);
Py_CLEAR(self->decoder);
if (self->buf)
PyMem_Free(self->buf);
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
Py_TYPE(self)->tp_free(self);
}
static PyObject *
stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
stringio *self;
assert(type != NULL && type->tp_alloc != NULL);
self = (stringio *)type->tp_alloc(type, 0);
if (self == NULL)
return NULL;
self->string_size = 0;
self->pos = 0;
self->buf_size = 0;
self->buf = (Py_UNICODE *)PyMem_Malloc(0);
if (self->buf == NULL) {
Py_DECREF(self);
return PyErr_NoMemory();
}
return (PyObject *)self;
}
static int
stringio_init(stringio *self, PyObject *args, PyObject *kwds)
{
char *kwlist[] = {"initial_value", "newline", NULL};
PyObject *value = NULL;
char *newline = "\n";
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oz:__init__", kwlist,
&value, &newline))
return -1;
if (newline && newline[0] != '\0'
&& !(newline[0] == '\n' && newline[1] == '\0')
&& !(newline[0] == '\r' && newline[1] == '\0')
&& !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) {
PyErr_Format(PyExc_ValueError,
"illegal newline value: %s", newline);
return -1;
}
if (value && value != Py_None && !PyUnicode_Check(value)) {
PyErr_Format(PyExc_ValueError,
"initial_value must be str or None, not %.200s",
Py_TYPE(value)->tp_name);
return -1;
}
self->ok = 0;
Py_CLEAR(self->readnl);
Py_CLEAR(self->writenl);
Py_CLEAR(self->decoder);
if (newline) {
self->readnl = PyString_FromString(newline);
if (self->readnl == NULL)
return -1;
}
self->readuniversal = (newline == NULL || newline[0] == '\0');
self->readtranslate = (newline == NULL);
/* If newline == "", we don't translate anything.
If newline == "\n" or newline == None, we translate to "\n", which is
a no-op.
(for newline == None, TextIOWrapper translates to os.sepline, but it
is pointless for StringIO)
*/
if (newline != NULL && newline[0] == '\r') {
self->writenl = PyUnicode_FromString(newline);
}
if (self->readuniversal) {
self->decoder = PyObject_CallFunction(
(PyObject *)&PyIncrementalNewlineDecoder_Type,
"Oi", Py_None, (int) self->readtranslate);
if (self->decoder == NULL)
return -1;
}
/* Now everything is set up, resize buffer to size of initial value,
and copy it */
self->string_size = 0;
if (value && value != Py_None) {
Py_ssize_t len = PyUnicode_GetSize(value);
/* This is a heuristic, for newline translation might change
the string length. */
if (resize_buffer(self, len) < 0)
return -1;
self->pos = 0;
if (write_str(self, value) < 0)
return -1;
}
else {
if (resize_buffer(self, 0) < 0)
return -1;
}
self->pos = 0;
self->closed = 0;
self->ok = 1;
return 0;
}
/* Properties and pseudo-properties */
static PyObject *
stringio_seekable(stringio *self, PyObject *args)
{
CHECK_INITIALIZED(self);
Py_RETURN_TRUE;
}
static PyObject *
stringio_readable(stringio *self, PyObject *args)
{
CHECK_INITIALIZED(self);
Py_RETURN_TRUE;
}
static PyObject *
stringio_writable(stringio *self, PyObject *args)
{
CHECK_INITIALIZED(self);
Py_RETURN_TRUE;
}
static PyObject *
stringio_buffer(stringio *self, void *context)
{
PyErr_SetString(_PyIO_unsupported_operation,
"buffer attribute is unsupported on type StringIO");
return NULL;
}
static PyObject *
stringio_closed(stringio *self, void *context)
{
CHECK_INITIALIZED(self);
return PyBool_FromLong(self->closed);
}
static PyObject *
stringio_line_buffering(stringio *self, void *context)
{
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
Py_RETURN_FALSE;
}
static PyObject *
stringio_newlines(stringio *self, void *context)
{
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
if (self->decoder == NULL)
Py_RETURN_NONE;
return PyObject_GetAttr(self->decoder, _PyIO_str_newlines);
}
static struct PyMethodDef stringio_methods[] = {
{"close", (PyCFunction)stringio_close, METH_NOARGS, stringio_close_doc},
{"getvalue", (PyCFunction)stringio_getvalue, METH_VARARGS, stringio_getvalue_doc},
{"read", (PyCFunction)stringio_read, METH_VARARGS, stringio_read_doc},
{"readline", (PyCFunction)stringio_readline, METH_VARARGS, stringio_readline_doc},
{"tell", (PyCFunction)stringio_tell, METH_NOARGS, stringio_tell_doc},
{"truncate", (PyCFunction)stringio_truncate, METH_VARARGS, stringio_truncate_doc},
{"seek", (PyCFunction)stringio_seek, METH_VARARGS, stringio_seek_doc},
{"write", (PyCFunction)stringio_write, METH_O, stringio_write_doc},
{"seekable", (PyCFunction)stringio_seekable, METH_NOARGS},
{"readable", (PyCFunction)stringio_readable, METH_NOARGS},
{"writable", (PyCFunction)stringio_writable, METH_NOARGS},
{NULL, NULL} /* sentinel */
};
static PyGetSetDef stringio_getset[] = {
{"closed", (getter)stringio_closed, NULL, NULL},
{"newlines", (getter)stringio_newlines, NULL, NULL},
/* (following comments straight off of the original Python wrapper:)
XXX Cruft to support the TextIOWrapper API. This would only
be meaningful if StringIO supported the buffer attribute.
Hopefully, a better solution, than adding these pseudo-attributes,
will be found.
*/
{"buffer", (getter)stringio_buffer, NULL, NULL},
{"line_buffering", (getter)stringio_line_buffering, NULL, NULL},
{NULL}
};
PyTypeObject PyStringIO_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
"_io.StringIO", /*tp_name*/
sizeof(stringio), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)stringio_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_reserved*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash*/
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
stringio_doc, /*tp_doc*/
(traverseproc)stringio_traverse, /*tp_traverse*/
(inquiry)stringio_clear, /*tp_clear*/
0, /*tp_richcompare*/
offsetof(stringio, weakreflist), /*tp_weaklistoffset*/
0, /*tp_iter*/
(iternextfunc)stringio_iternext, /*tp_iternext*/
stringio_methods, /*tp_methods*/
0, /*tp_members*/
stringio_getset, /*tp_getset*/
0, /*tp_base*/
0, /*tp_dict*/
0, /*tp_descr_get*/
0, /*tp_descr_set*/
offsetof(stringio, dict), /*tp_dictoffset*/
(initproc)stringio_init, /*tp_init*/
0, /*tp_alloc*/
stringio_new, /*tp_new*/
};

2606
Modules/_io/textio.c Normal file
View File

File diff suppressed because it is too large Load Diff

9
setup.py
View File

@ -438,10 +438,11 @@ class PyBuildExt(build_ext):
exts.append( Extension("_heapq", ["_heapqmodule.c"]) )
# operator.add() and similar goodies
exts.append( Extension('operator', ['operator.c']) )
# Python 3.0 _fileio module
exts.append( Extension("_fileio", ["_fileio.c"]) )
# Python 3.0 _bytesio module
exts.append( Extension("_bytesio", ["_bytesio.c"]) )
# Python 3.1 _io library
exts.append( Extension("_io",
["_io/bufferedio.c", "_io/bytesio.c", "_io/fileio.c",
"_io/iobase.c", "_io/_iomodule.c", "_io/stringio.c", "_io/textio.c"],
depends=["_io/_iomodule.h"], include_dirs=["Modules/_io"]))
# _functools
exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
# _json speedups