Issue #6215: backport the 3.1 io lib

Doc/library/io.rst

@@ -573,6 +573,10 @@ Text I/O
       The name of the encoding used to decode the stream's bytes into
       strings, and to encode strings into bytes.
 
+   .. attribute:: errors
+
+      The error setting of the decoder or encoder.
+
    .. attribute:: newlines
 
       A string, a tuple of strings, or ``None``, indicating the newlines
@@ -625,13 +629,9 @@ Text I/O
    If *line_buffering* is ``True``, :meth:`flush` is implied when a call to
    write contains a newline character.
 
-   :class:`TextIOWrapper` provides these data attributes in addition to those of
+   :class:`TextIOWrapper` provides one attribute in addition to those of
    :class:`TextIOBase` and its parents:
 
-   .. attribute:: errors
-
-      The encoding and decoding error setting.
-
    .. attribute:: line_buffering
 
       Whether line buffering is enabled.

Lib/test/test_bufio.py

@@ -1,11 +1,14 @@
 import unittest
-from test import test_support
+from test import test_support as support
 
-# Simple test to ensure that optimizations in fileobject.c deliver
+import io # C implementation.
-# the expected results.  For best testing, run this under a debug-build
+import _pyio as pyio # Python implementation.
-# Python too (to exercise asserts in the C code).
 
-lengths = range(1, 257) + [512, 1000, 1024, 2048, 4096, 8192, 10000,
+# Simple test to ensure that optimizations in the IO library deliver the
+# expected results.  For best testing, run this under a debug-build Python too
+# (to exercise asserts in the C code).
+
+lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000,
                                  16384, 32768, 65536, 1000000]
 
 class BufferSizeTest(unittest.TestCase):
@@ -14,27 +17,27 @@ class BufferSizeTest(unittest.TestCase):
         # .readline()s deliver what we wrote.
 
         # Ensure we can open TESTFN for writing.
-        test_support.unlink(test_support.TESTFN)
+        support.unlink(support.TESTFN)
 
         # Since C doesn't guarantee we can write/read arbitrary bytes in text
         # files, use binary mode.
-        f = open(test_support.TESTFN, "wb")
+        f = self.open(support.TESTFN, "wb")
         try:
             # write once with \n and once without
             f.write(s)
-            f.write("\n")
+            f.write(b"\n")
             f.write(s)
             f.close()
-            f = open(test_support.TESTFN, "rb")
+            f = open(support.TESTFN, "rb")
             line = f.readline()
-            self.assertEqual(line, s + "\n")
+            self.assertEqual(line, s + b"\n")
             line = f.readline()
             self.assertEqual(line, s)
             line = f.readline()
             self.assert_(not line) # Must be at EOF
             f.close()
         finally:
-            test_support.unlink(test_support.TESTFN)
+            support.unlink(support.TESTFN)
 
     def drive_one(self, pattern):
         for length in lengths:
@@ -47,19 +50,27 @@ class BufferSizeTest(unittest.TestCase):
             teststring = pattern * q + pattern[:r]
             self.assertEqual(len(teststring), length)
             self.try_one(teststring)
-            self.try_one(teststring + "x")
+            self.try_one(teststring + b"x")
             self.try_one(teststring[:-1])
 
     def test_primepat(self):
         # A pattern with prime length, to avoid simple relationships with
         # stdio buffer sizes.
-        self.drive_one("1234567890\00\01\02\03\04\05\06")
+        self.drive_one(b"1234567890\00\01\02\03\04\05\06")
 
     def test_nullpat(self):
-        self.drive_one("\0" * 1000)
+        self.drive_one(bytes(1000))
+
+
+class CBufferSizeTest(BufferSizeTest):
+    open = io.open
+
+class PyBufferSizeTest(BufferSizeTest):
+    open = staticmethod(pyio.open)
+
 
 def test_main():
-    test_support.run_unittest(BufferSizeTest)
+    support.run_unittest(CBufferSizeTest, PyBufferSizeTest)
 
 if __name__ == "__main__":
     test_main()

Lib/test/test_file.py

@@ -1,13 +1,14 @@
+from __future__ import print_function
+
 import sys
 import os
 import unittest
-import itertools
-import time
-import threading
 from array import array
 from weakref import proxy
 
-from test import test_support
+import io
+import _pyio as pyio
+
 from test.test_support import TESTFN, findfile, run_unittest
 from UserList import UserList
 
@@ -15,7 +16,7 @@ class AutoFileTests(unittest.TestCase):
     # file tests for which a test file is automatically set up
 
     def setUp(self):
-        self.f = open(TESTFN, 'wb')
+        self.f = self.open(TESTFN, 'wb')
 
     def tearDown(self):
         if self.f:
@@ -25,7 +26,7 @@ class AutoFileTests(unittest.TestCase):
     def testWeakRefs(self):
         # verify weak references
         p = proxy(self.f)
-        p.write('teststring')
+        p.write(b'teststring')
         self.assertEquals(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
@@ -34,35 +35,35 @@ class AutoFileTests(unittest.TestCase):
     def testAttributes(self):
         # verify expected attributes exist
         f = self.f
-        softspace = f.softspace
         f.name     # merely shouldn't blow up
         f.mode     # ditto
         f.closed   # ditto
 
-        # verify softspace is writable
-        f.softspace = softspace    # merely shouldn't blow up
-
-        # verify the others aren't
-        for attr in 'name', 'mode', 'closed':
-            self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops')
-
     def testReadinto(self):
         # verify readinto
-        self.f.write('12')
+        self.f.write(b'12')
         self.f.close()
-        a = array('c', 'x'*10)
+        a = array('b', b'x'*10)
-        self.f = open(TESTFN, 'rb')
+        self.f = self.open(TESTFN, 'rb')
         n = self.f.readinto(a)
-        self.assertEquals('12', a.tostring()[:n])
+        self.assertEquals(b'12', a.tostring()[:n])
+
+    def testReadinto_text(self):
+        # verify readinto refuses text files
+        a = array('b', b'x'*10)
+        self.f.close()
+        self.f = self.open(TESTFN, 'r')
+        if hasattr(self.f, "readinto"):
+            self.assertRaises(TypeError, self.f.readinto, a)
 
     def testWritelinesUserList(self):
         # verify writelines with instance sequence
-        l = UserList(['1', '2'])
+        l = UserList([b'1', b'2'])
         self.f.writelines(l)
         self.f.close()
-        self.f = open(TESTFN, 'rb')
+        self.f = self.open(TESTFN, 'rb')
         buf = self.f.read()
-        self.assertEquals(buf, '12')
+        self.assertEquals(buf, b'12')
 
     def testWritelinesIntegers(self):
         # verify writelines with integers
@@ -81,36 +82,43 @@ class AutoFileTests(unittest.TestCase):
         self.assertRaises(TypeError, self.f.writelines,
                           [NonString(), NonString()])
 
-    def testRepr(self):
-        # verify repr works
-        self.assert_(repr(self.f).startswith("<open file '" + TESTFN))
-
     def testErrors(self):
         f = self.f
         self.assertEquals(f.name, TESTFN)
         self.assert_(not f.isatty())
         self.assert_(not f.closed)
 
-        self.assertRaises(TypeError, f.readinto, "")
+        if hasattr(f, "readinto"):
+            self.assertRaises((IOError, TypeError), f.readinto, "")
         f.close()
         self.assert_(f.closed)
 
     def testMethods(self):
-        methods = ['fileno', 'flush', 'isatty', 'next', 'read', 'readinto',
+        methods = [('fileno', ()),
-                   'readline', 'readlines', 'seek', 'tell', 'truncate',
+                   ('flush', ()),
-                   'write', 'xreadlines', '__iter__']
+                   ('isatty', ()),
-        if sys.platform.startswith('atheos'):
+                   ('next', ()),
-            methods.remove('truncate')
+                   ('read', ()),
+                   ('write', (b"",)),
+                   ('readline', ()),
+                   ('readlines', ()),
+                   ('seek', (0,)),
+                   ('tell', ()),
+                   ('write', (b"",)),
+                   ('writelines', ([],)),
+                   ('__iter__', ()),
+                   ]
+        if not sys.platform.startswith('atheos'):
+            methods.append(('truncate', ()))
 
         # __exit__ should close the file
         self.f.__exit__(None, None, None)
         self.assert_(self.f.closed)
 
-        for methodname in methods:
+        for methodname, args in methods:
             method = getattr(self.f, methodname)
             # should raise on closed file
-            self.assertRaises(ValueError, method)
+            self.assertRaises(ValueError, method, *args)
-        self.assertRaises(ValueError, self.f.writelines, [])
 
         # file is closed, __exit__ shouldn't do anything
         self.assertEquals(self.f.__exit__(None, None, None), None)
@@ -123,70 +131,47 @@ class AutoFileTests(unittest.TestCase):
     def testReadWhenWriting(self):
         self.assertRaises(IOError, self.f.read)
 
-class OtherFileTests(unittest.TestCase):
+class CAutoFileTests(AutoFileTests):
+    open = io.open
 
-    def testOpenDir(self):
+class PyAutoFileTests(AutoFileTests):
-        this_dir = os.path.dirname(__file__)
+    open = staticmethod(pyio.open)
-        for mode in (None, "w"):
+
-            try:
+
-                if mode:
+class OtherFileTests(unittest.TestCase):
-                    f = open(this_dir, mode)
-                else:
-                    f = open(this_dir)
-            except IOError as e:
-                self.assertEqual(e.filename, this_dir)
-            else:
-                self.fail("opening a directory didn't raise an IOError")
 
     def testModeStrings(self):
         # check invalid mode strings
         for mode in ("", "aU", "wU+"):
             try:
-                f = open(TESTFN, mode)
+                f = self.open(TESTFN, mode)
             except ValueError:
                 pass
             else:
                 f.close()
                 self.fail('%r is an invalid file mode' % mode)
 
-        # Some invalid modes fail on Windows, but pass on Unix
-        # Issue3965: avoid a crash on Windows when filename is unicode
-        for name in (TESTFN, unicode(TESTFN), unicode(TESTFN + '\t')):
-            try:
-                f = open(name, "rr")
-            except (IOError, ValueError):
-                pass
-            else:
-                f.close()
-
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises(IOError, sys.stdin.seek, -1)
+            self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
         else:
-            print >>sys.__stdout__, (
+            print((
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
-                ' Test manually.')
+                ' Test manually.'), file=sys.__stdout__)
-        self.assertRaises(IOError, sys.stdin.truncate)
+        self.assertRaises((IOError, ValueError), sys.stdin.truncate)
-
-    def testUnicodeOpen(self):
-        # verify repr works for unicode too
-        f = open(unicode(TESTFN), "w")
-        self.assert_(repr(f).startswith("<open file u'" + TESTFN))
-        f.close()
-        os.unlink(TESTFN)
 
     def testBadModeArgument(self):
         # verify that we get a sensible error message for bad mode argument
         bad_mode = "qwerty"
         try:
-            f = open(TESTFN, bad_mode)
+            f = self.open(TESTFN, bad_mode)
-        except ValueError, msg:
+        except ValueError as msg:
-            if msg[0] != 0:
+            if msg.args[0] != 0:
                 s = str(msg)
                 if s.find(TESTFN) != -1 or s.find(bad_mode) == -1:
                     self.fail("bad error message for invalid mode: %s" % s)
-            # if msg[0] == 0, we're probably on Windows where there may be
+            # if msg.args[0] == 0, we're probably on Windows where there may be
             # no obvious way to discover why open() failed.
         else:
             f.close()
@@ -197,31 +182,32 @@ class OtherFileTests(unittest.TestCase):
         # misbehaviour especially with repeated close() calls
         for s in (-1, 0, 1, 512):
             try:
-                f = open(TESTFN, 'w', s)
+                f = self.open(TESTFN, 'wb', s)
-                f.write(str(s))
+                f.write(str(s).encode("ascii"))
                 f.close()
                 f.close()
-                f = open(TESTFN, 'r', s)
+                f = self.open(TESTFN, 'rb', s)
-                d = int(f.read())
+                d = int(f.read().decode("ascii"))
                 f.close()
                 f.close()
-            except IOError, msg:
+            except IOError as msg:
                 self.fail('error setting buffer size %d: %s' % (s, str(msg)))
             self.assertEquals(d, s)
 
     def testTruncateOnWindows(self):
-        os.unlink(TESTFN)
-
-        def bug801631():
         # SF bug <http://www.python.org/sf/801631>
         # "file.truncate fault on windows"
-            f = open(TESTFN, 'wb')
+
-            f.write('12345678901')   # 11 bytes
+        os.unlink(TESTFN)
+        f = self.open(TESTFN, 'wb')
+
+        try:
+            f.write(b'12345678901')   # 11 bytes
             f.close()
 
-            f = open(TESTFN,'rb+')
+            f = self.open(TESTFN,'rb+')
             data = f.read(5)
-            if data != '12345':
+            if data != b'12345':
                 self.fail("Read on file opened for update failed %r" % data)
             if f.tell() != 5:
                 self.fail("File pos after read wrong %d" % f.tell())
@@ -234,56 +220,42 @@ class OtherFileTests(unittest.TestCase):
             size = os.path.getsize(TESTFN)
             if size != 5:
                 self.fail("File size after ftruncate wrong %d" % size)
-
-        try:
-            bug801631()
         finally:
+            f.close()
             os.unlink(TESTFN)
 
     def testIteration(self):
         # Test the complex interaction when mixing file-iteration and the
-        # various read* methods. Ostensibly, the mixture could just be tested
+        # various read* methods.
-        # to work when it should work according to the Python language,
-        # instead of fail when it should fail according to the current CPython
-        # implementation.  People don't always program Python the way they
-        # should, though, and the implemenation might change in subtle ways,
-        # so we explicitly test for errors, too; the test will just have to
-        # be updated when the implementation changes.
         dataoffset = 16384
-        filler = "ham\n"
+        filler = b"ham\n"
         assert not dataoffset % len(filler), \
             "dataoffset must be multiple of len(filler)"
         nchunks = dataoffset // len(filler)
         testlines = [
-            "spam, spam and eggs\n",
+            b"spam, spam and eggs\n",
-            "eggs, spam, ham and spam\n",
+            b"eggs, spam, ham and spam\n",
-            "saussages, spam, spam and eggs\n",
+            b"saussages, spam, spam and eggs\n",
-            "spam, ham, spam and eggs\n",
+            b"spam, ham, spam and eggs\n",
-            "spam, spam, spam, spam, spam, ham, spam\n",
+            b"spam, spam, spam, spam, spam, ham, spam\n",
-            "wonderful spaaaaaam.\n"
+            b"wonderful spaaaaaam.\n"
         ]
         methods = [("readline", ()), ("read", ()), ("readlines", ()),
-                   ("readinto", (array("c", " "*100),))]
+                   ("readinto", (array("b", b" "*100),))]
 
         try:
             # Prepare the testfile
-            bag = open(TESTFN, "w")
+            bag = self.open(TESTFN, "wb")
             bag.write(filler * nchunks)
             bag.writelines(testlines)
             bag.close()
             # Test for appropriate errors mixing read* and iteration
             for methodname, args in methods:
-                f = open(TESTFN)
+                f = self.open(TESTFN, 'rb')
-                if f.next() != filler:
+                if next(f) != filler:
                     self.fail, "Broken testfile"
                 meth = getattr(f, methodname)
-                try:
+                meth(*args)  # This simply shouldn't fail
-                    meth(*args)
-                except ValueError:
-                    pass
-                else:
-                    self.fail("%s%r after next() didn't raise ValueError" %
-                                     (methodname, args))
                 f.close()
 
             # Test to see if harmless (by accident) mixing of read* and
@@ -293,9 +265,9 @@ class OtherFileTests(unittest.TestCase):
             # ("h", "a", "m", "\n"), so 4096 lines of that should get us
             # exactly on the buffer boundary for any power-of-2 buffersize
             # between 4 and 16384 (inclusive).
-            f = open(TESTFN)
+            f = self.open(TESTFN, 'rb')
             for i in range(nchunks):
-                f.next()
+                next(f)
             testline = testlines.pop(0)
             try:
                 line = f.readline()
@@ -306,7 +278,7 @@ class OtherFileTests(unittest.TestCase):
                 self.fail("readline() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
             testline = testlines.pop(0)
-            buf = array("c", "\x00" * len(testline))
+            buf = array("b", b"\x00" * len(testline))
             try:
                 f.readinto(buf)
             except ValueError:
@@ -335,7 +307,7 @@ class OtherFileTests(unittest.TestCase):
                 self.fail("readlines() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
             # Reading after iteration hit EOF shouldn't hurt either
-            f = open(TESTFN)
+            f = self.open(TESTFN, 'rb')
             try:
                 for line in f:
                     pass
@@ -351,222 +323,19 @@ class OtherFileTests(unittest.TestCase):
         finally:
             os.unlink(TESTFN)
 
-class FileSubclassTests(unittest.TestCase):
+class COtherFileTests(OtherFileTests):
+    open = io.open
 
-    def testExit(self):
+class PyOtherFileTests(OtherFileTests):
-        # test that exiting with context calls subclass' close
+    open = staticmethod(pyio.open)
-        class C(file):
-            def __init__(self, *args):
-                self.subclass_closed = False
-                file.__init__(self, *args)
-            def close(self):
-                self.subclass_closed = True
-                file.close(self)
-
-        with C(TESTFN, 'w') as f:
-            pass
-        self.failUnless(f.subclass_closed)
-
-
-class FileThreadingTests(unittest.TestCase):
-    # These tests check the ability to call various methods of file objects
-    # (including close()) concurrently without crashing the Python interpreter.
-    # See #815646, #595601
-
-    def setUp(self):
-        self.f = None
-        self.filename = TESTFN
-        with open(self.filename, "w") as f:
-            f.write("\n".join("0123456789"))
-        self._count_lock = threading.Lock()
-        self.close_count = 0
-        self.close_success_count = 0
-
-    def tearDown(self):
-        if self.f:
-            try:
-                self.f.close()
-            except (EnvironmentError, ValueError):
-                pass
-        try:
-            os.remove(self.filename)
-        except EnvironmentError:
-            pass
-
-    def _create_file(self):
-        self.f = open(self.filename, "w+")
-
-    def _close_file(self):
-        with self._count_lock:
-            self.close_count += 1
-        self.f.close()
-        with self._count_lock:
-            self.close_success_count += 1
-
-    def _close_and_reopen_file(self):
-        self._close_file()
-        # if close raises an exception thats fine, self.f remains valid so
-        # we don't need to reopen.
-        self._create_file()
-
-    def _run_workers(self, func, nb_workers, duration=0.2):
-        with self._count_lock:
-            self.close_count = 0
-            self.close_success_count = 0
-        self.do_continue = True
-        threads = []
-        try:
-            for i in range(nb_workers):
-                t = threading.Thread(target=func)
-                t.start()
-                threads.append(t)
-            for _ in xrange(100):
-                time.sleep(duration/100)
-                with self._count_lock:
-                    if self.close_count-self.close_success_count > nb_workers+1:
-                        if test_support.verbose:
-                            print 'Q',
-                        break
-            time.sleep(duration)
-        finally:
-            self.do_continue = False
-            for t in threads:
-                t.join()
-
-    def _test_close_open_io(self, io_func, nb_workers=5):
-        def worker():
-            self._create_file()
-            funcs = itertools.cycle((
-                lambda: io_func(),
-                lambda: self._close_and_reopen_file(),
-            ))
-            for f in funcs:
-                if not self.do_continue:
-                    break
-                try:
-                    f()
-                except (IOError, ValueError):
-                    pass
-        self._run_workers(worker, nb_workers)
-        if test_support.verbose:
-            # Useful verbose statistics when tuning this test to take
-            # less time to run but still ensuring that its still useful.
-            #
-            # the percent of close calls that raised an error
-            percent = 100. - 100.*self.close_success_count/self.close_count
-            print self.close_count, ('%.4f ' % percent),
-
-    def test_close_open(self):
-        def io_func():
-            pass
-        self._test_close_open_io(io_func)
-
-    def test_close_open_flush(self):
-        def io_func():
-            self.f.flush()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_iter(self):
-        def io_func():
-            list(iter(self.f))
-        self._test_close_open_io(io_func)
-
-    def test_close_open_isatty(self):
-        def io_func():
-            self.f.isatty()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_print(self):
-        def io_func():
-            print >> self.f, ''
-        self._test_close_open_io(io_func)
-
-    def test_close_open_read(self):
-        def io_func():
-            self.f.read(0)
-        self._test_close_open_io(io_func)
-
-    def test_close_open_readinto(self):
-        def io_func():
-            a = array('c', 'xxxxx')
-            self.f.readinto(a)
-        self._test_close_open_io(io_func)
-
-    def test_close_open_readline(self):
-        def io_func():
-            self.f.readline()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_readlines(self):
-        def io_func():
-            self.f.readlines()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_seek(self):
-        def io_func():
-            self.f.seek(0, 0)
-        self._test_close_open_io(io_func)
-
-    def test_close_open_tell(self):
-        def io_func():
-            self.f.tell()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_truncate(self):
-        def io_func():
-            self.f.truncate()
-        self._test_close_open_io(io_func)
-
-    def test_close_open_write(self):
-        def io_func():
-            self.f.write('')
-        self._test_close_open_io(io_func)
-
-    def test_close_open_writelines(self):
-        def io_func():
-            self.f.writelines('')
-        self._test_close_open_io(io_func)
-
-
-class StdoutTests(unittest.TestCase):
-
-    def test_move_stdout_on_write(self):
-        # Issue 3242: sys.stdout can be replaced (and freed) during a
-        # print statement; prevent a segfault in this case
-        save_stdout = sys.stdout
-
-        class File:
-            def write(self, data):
-                if '\n' in data:
-                    sys.stdout = save_stdout
-
-        try:
-            sys.stdout = File()
-            print "some text"
-        finally:
-            sys.stdout = save_stdout
-
-    def test_del_stdout_before_print(self):
-        # Issue 4597: 'print' with no argument wasn't reporting when
-        # sys.stdout was deleted.
-        save_stdout = sys.stdout
-        del sys.stdout
-        try:
-            print
-        except RuntimeError as e:
-            self.assertEquals(str(e), "lost sys.stdout")
-        else:
-            self.fail("Expected RuntimeError")
-        finally:
-            sys.stdout = save_stdout
 
 
 def test_main():
     # Historically, these tests have been sloppy about removing TESTFN.
     # So get rid of it no matter what.
     try:
-        run_unittest(AutoFileTests, OtherFileTests, FileSubclassTests,
+        run_unittest(CAutoFileTests, PyAutoFileTests,
-            FileThreadingTests, StdoutTests)
+                     COtherFileTests, PyOtherFileTests)
     finally:
         if os.path.exists(TESTFN):
             os.unlink(TESTFN)

Lib/test/test_fileio.py

@@ -1,23 +1,26 @@
 # Adapted from test_file.py by Daniel Stutzbach
-#from __future__ import unicode_literals
+
+from __future__ import unicode_literals
 
 import sys
 import os
+import errno
 import unittest
 from array import array
 from weakref import proxy
+from functools import wraps
 
 from test.test_support import (TESTFN, findfile, check_warnings, run_unittest,
                                make_bad_fd)
-from UserList import UserList
+from test.test_support import py3k_bytes as bytes
 
-import _fileio
+from _io import FileIO as _FileIO
 
 class AutoFileTests(unittest.TestCase):
     # file tests for which a test file is automatically set up
 
     def setUp(self):
-        self.f = _fileio._FileIO(TESTFN, 'w')
+        self.f = _FileIO(TESTFN, 'w')
 
     def tearDown(self):
         if self.f:
@@ -34,7 +37,7 @@ class AutoFileTests(unittest.TestCase):
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testSeekTell(self):
-        self.f.write(bytes(bytearray(range(20))))
+        self.f.write(bytes(range(20)))
         self.assertEquals(self.f.tell(), 20)
         self.f.seek(0)
         self.assertEquals(self.f.tell(), 0)
@@ -61,17 +64,21 @@ class AutoFileTests(unittest.TestCase):
 
     def testReadinto(self):
         # verify readinto
-        self.f.write(bytes(bytearray([1, 2])))
+        self.f.write(b"\x01\x02")
         self.f.close()
-        a = array('b', b'x'*10)
+        a = array(b'b', b'x'*10)
-        self.f = _fileio._FileIO(TESTFN, 'r')
+        self.f = _FileIO(TESTFN, 'r')
         n = self.f.readinto(a)
-        self.assertEquals(array('b', [1, 2]), a[:n])
+        self.assertEquals(array(b'b', [1, 2]), a[:n])
 
     def testRepr(self):
-        self.assertEquals(repr(self.f),
+        self.assertEquals(repr(self.f), "<_io.FileIO name=%r mode='%s'>"
-                          "_fileio._FileIO(%d, %s)" % (self.f.fileno(),
+                                        % (self.f.name, self.f.mode))
-                                                       repr(self.f.mode)))
+        del self.f.name
+        self.assertEquals(repr(self.f), "<_io.FileIO fd=%r mode='%s'>"
+                                        % (self.f.fileno(), self.f.mode))
+        self.f.close()
+        self.assertEquals(repr(self.f), "<_io.FileIO [closed]>")
 
     def testErrors(self):
         f = self.f
@@ -81,7 +88,7 @@ class AutoFileTests(unittest.TestCase):
         self.assertRaises(ValueError, f.read, 10) # Open for reading
         f.close()
         self.assert_(f.closed)
-        f = _fileio._FileIO(TESTFN, 'r')
+        f = _FileIO(TESTFN, 'r')
         self.assertRaises(TypeError, f.readinto, "")
         self.assert_(not f.closed)
         f.close()
@@ -107,31 +114,131 @@ class AutoFileTests(unittest.TestCase):
         # Windows always returns "[Errno 13]: Permission denied
         # Unix calls dircheck() and returns "[Errno 21]: Is a directory"
         try:
-            _fileio._FileIO('.', 'r')
+            _FileIO('.', 'r')
         except IOError as e:
             self.assertNotEqual(e.errno, 0)
             self.assertEqual(e.filename, ".")
         else:
             self.fail("Should have raised IOError")
 
+    #A set of functions testing that we get expected behaviour if someone has
+    #manually closed the internal file descriptor.  First, a decorator:
+    def ClosedFD(func):
+        @wraps(func)
+        def wrapper(self):
+            #forcibly close the fd before invoking the problem function
+            f = self.f
+            os.close(f.fileno())
+            try:
+                func(self, f)
+            finally:
+                try:
+                    self.f.close()
+                except IOError:
+                    pass
+        return wrapper
+
+    def ClosedFDRaises(func):
+        @wraps(func)
+        def wrapper(self):
+            #forcibly close the fd before invoking the problem function
+            f = self.f
+            os.close(f.fileno())
+            try:
+                func(self, f)
+            except IOError as e:
+                self.assertEqual(e.errno, errno.EBADF)
+            else:
+                self.fail("Should have raised IOError")
+            finally:
+                try:
+                    self.f.close()
+                except IOError:
+                    pass
+        return wrapper
+
+    @ClosedFDRaises
+    def testErrnoOnClose(self, f):
+        f.close()
+
+    @ClosedFDRaises
+    def testErrnoOnClosedWrite(self, f):
+        f.write('a')
+
+    @ClosedFDRaises
+    def testErrnoOnClosedSeek(self, f):
+        f.seek(0)
+
+    @ClosedFDRaises
+    def testErrnoOnClosedTell(self, f):
+        f.tell()
+
+    @ClosedFDRaises
+    def testErrnoOnClosedTruncate(self, f):
+        f.truncate(0)
+
+    @ClosedFD
+    def testErrnoOnClosedSeekable(self, f):
+        f.seekable()
+
+    @ClosedFD
+    def testErrnoOnClosedReadable(self, f):
+        f.readable()
+
+    @ClosedFD
+    def testErrnoOnClosedWritable(self, f):
+        f.writable()
+
+    @ClosedFD
+    def testErrnoOnClosedFileno(self, f):
+        f.fileno()
+
+    @ClosedFD
+    def testErrnoOnClosedIsatty(self, f):
+        self.assertEqual(f.isatty(), False)
+
+    def ReopenForRead(self):
+        try:
+            self.f.close()
+        except IOError:
+            pass
+        self.f = _FileIO(TESTFN, 'r')
+        os.close(self.f.fileno())
+        return self.f
+
+    @ClosedFDRaises
+    def testErrnoOnClosedRead(self, f):
+        f = self.ReopenForRead()
+        f.read(1)
+
+    @ClosedFDRaises
+    def testErrnoOnClosedReadall(self, f):
+        f = self.ReopenForRead()
+        f.readall()
+
+    @ClosedFDRaises
+    def testErrnoOnClosedReadinto(self, f):
+        f = self.ReopenForRead()
+        a = array(b'b', b'x'*10)
+        f.readinto(a)
 
 class OtherFileTests(unittest.TestCase):
 
     def testAbles(self):
         try:
-            f = _fileio._FileIO(TESTFN, "w")
+            f = _FileIO(TESTFN, "w")
             self.assertEquals(f.readable(), False)
             self.assertEquals(f.writable(), True)
             self.assertEquals(f.seekable(), True)
             f.close()
 
-            f = _fileio._FileIO(TESTFN, "r")
+            f = _FileIO(TESTFN, "r")
             self.assertEquals(f.readable(), True)
             self.assertEquals(f.writable(), False)
             self.assertEquals(f.seekable(), True)
             f.close()
 
-            f = _fileio._FileIO(TESTFN, "a+")
+            f = _FileIO(TESTFN, "a+")
             self.assertEquals(f.readable(), True)
             self.assertEquals(f.writable(), True)
             self.assertEquals(f.seekable(), True)
@@ -140,14 +247,14 @@ class OtherFileTests(unittest.TestCase):
 
             if sys.platform != "win32":
                 try:
-                    f = _fileio._FileIO("/dev/tty", "a")
+                    f = _FileIO("/dev/tty", "a")
                 except EnvironmentError:
                     # When run in a cron job there just aren't any
                     # ttys, so skip the test.  This also handles other
                     # OS'es that don't support /dev/tty.
                     pass
                 else:
-                    f = _fileio._FileIO("/dev/tty", "a")
+                    f = _FileIO("/dev/tty", "a")
                     self.assertEquals(f.readable(), False)
                     self.assertEquals(f.writable(), True)
                     if sys.platform != "darwin" and \
@@ -164,7 +271,7 @@ class OtherFileTests(unittest.TestCase):
         # check invalid mode strings
         for mode in ("", "aU", "wU+", "rw", "rt"):
             try:
-                f = _fileio._FileIO(TESTFN, mode)
+                f = _FileIO(TESTFN, mode)
             except ValueError:
                 pass
             else:
@@ -173,19 +280,35 @@ class OtherFileTests(unittest.TestCase):
 
     def testUnicodeOpen(self):
         # verify repr works for unicode too
-        f = _fileio._FileIO(str(TESTFN), "w")
+        f = _FileIO(str(TESTFN), "w")
         f.close()
         os.unlink(TESTFN)
 
+    def testBytesOpen(self):
+        # Opening a bytes filename
+        try:
+            fn = TESTFN.encode("ascii")
+        except UnicodeEncodeError:
+            # Skip test
+            return
+        f = _FileIO(fn, "w")
+        try:
+            f.write(b"abc")
+            f.close()
+            with open(TESTFN, "rb") as f:
+                self.assertEquals(f.read(), b"abc")
+        finally:
+            os.unlink(TESTFN)
+
     def testInvalidFd(self):
-        self.assertRaises(ValueError, _fileio._FileIO, -10)
+        self.assertRaises(ValueError, _FileIO, -10)
-        self.assertRaises(OSError, _fileio._FileIO, make_bad_fd())
+        self.assertRaises(OSError, _FileIO, make_bad_fd())
 
     def testBadModeArgument(self):
         # verify that we get a sensible error message for bad mode argument
         bad_mode = "qwerty"
         try:
-            f = _fileio._FileIO(TESTFN, bad_mode)
+            f = _FileIO(TESTFN, bad_mode)
         except ValueError as msg:
             if msg.args[0] != 0:
                 s = str(msg)
@@ -201,13 +324,13 @@ class OtherFileTests(unittest.TestCase):
         def bug801631():
             # SF bug <http://www.python.org/sf/801631>
             # "file.truncate fault on windows"
-            f = _fileio._FileIO(TESTFN, 'w')
+            f = _FileIO(TESTFN, 'w')
-            f.write(bytes(bytearray(range(11))))
+            f.write(bytes(range(11)))
             f.close()
 
-            f = _fileio._FileIO(TESTFN,'r+')
+            f = _FileIO(TESTFN,'r+')
             data = f.read(5)
-            if data != bytes(bytearray(range(5))):
+            if data != bytes(range(5)):
                 self.fail("Read on file opened for update failed %r" % data)
             if f.tell() != 5:
                 self.fail("File pos after read wrong %d" % f.tell())
@@ -245,14 +368,14 @@ class OtherFileTests(unittest.TestCase):
                 pass
 
     def testInvalidInit(self):
-        self.assertRaises(TypeError, _fileio._FileIO, "1", 0, 0)
+        self.assertRaises(TypeError, _FileIO, "1", 0, 0)
 
     def testWarnings(self):
         with check_warnings() as w:
             self.assertEqual(w.warnings, [])
-            self.assertRaises(TypeError, _fileio._FileIO, [])
+            self.assertRaises(TypeError, _FileIO, [])
             self.assertEqual(w.warnings, [])
-            self.assertRaises(ValueError, _fileio._FileIO, "/some/invalid/name", "rt")
+            self.assertRaises(ValueError, _FileIO, "/some/invalid/name", "rt")
             self.assertEqual(w.warnings, [])
 
 

Lib/test/test_largefile.py

@@ -1,12 +1,16 @@
 """Test largefile support on system where this makes sense.
 """
 
+from __future__ import print_function
+
 import os
 import stat
 import sys
 import unittest
 from test.test_support import run_unittest, TESTFN, verbose, requires, \
                               unlink
+import io  # C implementation of io
+import _pyio as pyio # Python implementation of io
 
 try:
     import signal
@@ -18,10 +22,10 @@ except (ImportError, AttributeError):
     pass
 
 # create >2GB file (2GB = 2147483648 bytes)
-size = 2500000000L
+size = 2500000000
 
 
-class TestCase(unittest.TestCase):
+class LargeFileTest(unittest.TestCase):
     """Test that each file function works as expected for a large
     (i.e. > 2GB, do  we have to check > 4GB) files.
 
@@ -33,28 +37,28 @@ class TestCase(unittest.TestCase):
 
     def test_seek(self):
         if verbose:
-            print 'create large file via seek (may be sparse file) ...'
+            print('create large file via seek (may be sparse file) ...')
-        with open(TESTFN, 'wb') as f:
+        with self.open(TESTFN, 'wb') as f:
-            f.write('z')
+            f.write(b'z')
             f.seek(0)
             f.seek(size)
-            f.write('a')
+            f.write(b'a')
             f.flush()
             if verbose:
-                print 'check file size with os.fstat'
+                print('check file size with os.fstat')
             self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1)
 
     def test_osstat(self):
         if verbose:
-            print 'check file size with os.stat'
+            print('check file size with os.stat')
         self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
 
     def test_seek_read(self):
         if verbose:
-            print 'play around with seek() and read() with the built largefile'
+            print('play around with seek() and read() with the built largefile')
-        with open(TESTFN, 'rb') as f:
+        with self.open(TESTFN, 'rb') as f:
             self.assertEqual(f.tell(), 0)
-            self.assertEqual(f.read(1), 'z')
+            self.assertEqual(f.read(1), b'z')
             self.assertEqual(f.tell(), 1)
             f.seek(0)
             self.assertEqual(f.tell(), 0)
@@ -77,15 +81,15 @@ class TestCase(unittest.TestCase):
             f.seek(size)
             self.assertEqual(f.tell(), size)
             # the 'a' that was written at the end of file above
-            self.assertEqual(f.read(1), 'a')
+            self.assertEqual(f.read(1), b'a')
             f.seek(-size-1, 1)
-            self.assertEqual(f.read(1), 'z')
+            self.assertEqual(f.read(1), b'z')
             self.assertEqual(f.tell(), 1)
 
     def test_lseek(self):
         if verbose:
-            print 'play around with os.lseek() with the built largefile'
+            print('play around with os.lseek() with the built largefile')
-        with open(TESTFN, 'rb') as f:
+        with self.open(TESTFN, 'rb') as f:
             self.assertEqual(os.lseek(f.fileno(), 0, 0), 0)
             self.assertEqual(os.lseek(f.fileno(), 42, 0), 42)
             self.assertEqual(os.lseek(f.fileno(), 42, 1), 84)
@@ -95,16 +99,16 @@ class TestCase(unittest.TestCase):
             self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0)
             self.assertEqual(os.lseek(f.fileno(), size, 0), size)
             # the 'a' that was written at the end of file above
-            self.assertEqual(f.read(1), 'a')
+            self.assertEqual(f.read(1), b'a')
 
     def test_truncate(self):
         if verbose:
-            print 'try truncate'
+            print('try truncate')
-        with open(TESTFN, 'r+b') as f:
+        with self.open(TESTFN, 'r+b') as f:
             # this is already decided before start running the test suite
             # but we do it anyway for extra protection
             if not hasattr(f, 'truncate'):
-                raise unittest.SkipTest, "open().truncate() not available on this system"
+                raise unittest.SkipTest("open().truncate() not available on this system")
             f.seek(0, 2)
             # else we've lost track of the true size
             self.assertEqual(f.tell(), size+1)
@@ -120,17 +124,25 @@ class TestCase(unittest.TestCase):
             newsize -= 1
             f.seek(42)
             f.truncate(newsize)
-            self.assertEqual(f.tell(), 42)       # else pointer moved
-            f.seek(0, 2)
             self.assertEqual(f.tell(), newsize)  # else wasn't truncated
-
+            f.seek(0, 2)
+            self.assertEqual(f.tell(), newsize)
             # XXX truncate(larger than true size) is ill-defined
             # across platform; cut it waaaaay back
             f.seek(0)
             f.truncate(1)
-            self.assertEqual(f.tell(), 0)       # else pointer moved
+            self.assertEqual(f.tell(), 1)       # else pointer moved
+            f.seek(0)
             self.assertEqual(len(f.read()), 1)  # else wasn't truncated
 
+    def test_seekable(self):
+        # Issue #5016; seekable() can return False when the current position
+        # is negative when truncated to an int.
+        for pos in (2**31-1, 2**31, 2**31+1):
+            with self.open(TESTFN, 'rb') as f:
+                f.seek(pos)
+                self.assert_(f.seekable())
+
 
 def test_main():
     # On Windows and Mac OSX this test comsumes large resources; It
@@ -144,34 +156,39 @@ def test_main():
         # Only run if the current filesystem supports large files.
         # (Skip this test on Windows, since we now always support
         # large files.)
-        f = open(TESTFN, 'wb')
+        f = open(TESTFN, 'wb', buffering=0)
         try:
             # 2**31 == 2147483648
-            f.seek(2147483649L)
+            f.seek(2147483649)
             # Seeking is not enough of a test: you must write and
             # flush, too!
-            f.write("x")
+            f.write(b'x')
             f.flush()
         except (IOError, OverflowError):
             f.close()
             unlink(TESTFN)
-            raise unittest.SkipTest, "filesystem does not have largefile support"
+            raise unittest.SkipTest("filesystem does not have largefile support")
         else:
             f.close()
     suite = unittest.TestSuite()
+    for _open, prefix in [(io.open, 'C'), (pyio.open, 'Py')]:
+        class TestCase(LargeFileTest):
+            pass
+        TestCase.open = staticmethod(_open)
+        TestCase.__name__ = prefix + LargeFileTest.__name__
         suite.addTest(TestCase('test_seek'))
         suite.addTest(TestCase('test_osstat'))
         suite.addTest(TestCase('test_seek_read'))
         suite.addTest(TestCase('test_lseek'))
-    with open(TESTFN, 'w') as f:
+        with _open(TESTFN, 'wb') as f:
             if hasattr(f, 'truncate'):
                 suite.addTest(TestCase('test_truncate'))
+        suite.addTest(TestCase('test_seekable'))
         unlink(TESTFN)
     try:
         run_unittest(suite)
     finally:
         unlink(TESTFN)
 
-
 if __name__ == '__main__':
     test_main()

Lib/test/test_memoryio.py

@@ -4,23 +4,66 @@ BytesIO -- for bytes
 """
 
 from __future__ import unicode_literals
+from __future__ import print_function
 
 import unittest
-from test import test_support
+from test import test_support as support
 
 import io
+import _pyio as pyio
 import sys
-import array
 
-try:
+class MemorySeekTestMixin:
-    import _bytesio
+
-    has_c_implementation = True
+    def testInit(self):
-except ImportError:
+        buf = self.buftype("1234567890")
-    has_c_implementation = False
+        bytesIo = self.ioclass(buf)
+
+    def testRead(self):
+        buf = self.buftype("1234567890")
+        bytesIo = self.ioclass(buf)
+
+        self.assertEquals(buf[:1], bytesIo.read(1))
+        self.assertEquals(buf[1:5], bytesIo.read(4))
+        self.assertEquals(buf[5:], bytesIo.read(900))
+        self.assertEquals(self.EOF, bytesIo.read())
+
+    def testReadNoArgs(self):
+        buf = self.buftype("1234567890")
+        bytesIo = self.ioclass(buf)
+
+        self.assertEquals(buf, bytesIo.read())
+        self.assertEquals(self.EOF, bytesIo.read())
+
+    def testSeek(self):
+        buf = self.buftype("1234567890")
+        bytesIo = self.ioclass(buf)
+
+        bytesIo.read(5)
+        bytesIo.seek(0)
+        self.assertEquals(buf, bytesIo.read())
+
+        bytesIo.seek(3)
+        self.assertEquals(buf[3:], bytesIo.read())
+        self.assertRaises(TypeError, bytesIo.seek, 0.0)
+
+    def testTell(self):
+        buf = self.buftype("1234567890")
+        bytesIo = self.ioclass(buf)
+
+        self.assertEquals(0, bytesIo.tell())
+        bytesIo.seek(5)
+        self.assertEquals(5, bytesIo.tell())
+        bytesIo.seek(10000)
+        self.assertEquals(10000, bytesIo.tell())
 
 
 class MemoryTestMixin:
 
+    def test_detach(self):
+        buf = self.ioclass()
+        self.assertRaises(self.UnsupportedOperation, buf.detach)
+
     def write_ops(self, f, t):
         self.assertEqual(f.write(t("blah.")), 5)
         self.assertEqual(f.seek(0), 0)
@@ -151,7 +194,7 @@ class MemoryTestMixin:
         self.assertEqual(memio.readline(), self.EOF)
         memio.seek(0)
         self.assertEqual(type(memio.readline()), type(buf))
-        self.assertEqual(memio.readline(None), buf)
+        self.assertEqual(memio.readline(), buf)
         self.assertRaises(TypeError, memio.readline, '')
         memio.close()
         self.assertRaises(ValueError,  memio.readline)
@@ -197,7 +240,7 @@ class MemoryTestMixin:
         self.assertEqual(i, 10)
         memio = self.ioclass(buf * 2)
         memio.close()
-        self.assertRaises(ValueError, memio.next)
+        self.assertRaises(ValueError, next, memio)
 
     def test_getvalue(self):
         buf = self.buftype("1234567890")
@@ -299,11 +342,14 @@ class MemoryTestMixin:
         self.assertEqual(test2(), buf)
 
 
-class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
+class PyBytesIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
+
+    UnsupportedOperation = pyio.UnsupportedOperation
+
     @staticmethod
     def buftype(s):
         return s.encode("ascii")
-    ioclass = io._BytesIO
+    ioclass = pyio.BytesIO
     EOF = b""
 
     def test_read1(self):
@@ -333,7 +379,8 @@ class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
         self.assertEqual(memio.readinto(b), 0)
         self.assertEqual(b, b"")
         self.assertRaises(TypeError, memio.readinto, '')
-        a = array.array(b'b', map(ord, b"hello world"))
+        import array
+        a = array.array(b'b', b"hello world")
         memio = self.ioclass(buf)
         memio.readinto(a)
         self.assertEqual(a.tostring(), b"1234567890d")
@@ -365,19 +412,41 @@ class PyBytesIOTest(MemoryTestMixin, unittest.TestCase):
 
     def test_bytes_array(self):
         buf = b"1234567890"
-
+        import array
-        a = array.array(b'b', map(ord, buf))
+        a = array.array(b'b', buf)
         memio = self.ioclass(a)
         self.assertEqual(memio.getvalue(), buf)
         self.assertEqual(memio.write(a), 10)
         self.assertEqual(memio.getvalue(), buf)
 
 
-class PyStringIOTest(MemoryTestMixin, unittest.TestCase):
+class PyStringIOTest(MemoryTestMixin, MemorySeekTestMixin, unittest.TestCase):
     buftype = unicode
-    ioclass = io.StringIO
+    ioclass = pyio.StringIO
+    UnsupportedOperation = pyio.UnsupportedOperation
     EOF = ""
 
+    # TextIO-specific behaviour.
+
+    def test_newlines_property(self):
+        memio = self.ioclass(newline=None)
+        # The C StringIO decodes newlines in write() calls, but the Python
+        # implementation only does when reading.  This function forces them to
+        # be decoded for testing.
+        def force_decode():
+            memio.seek(0)
+            memio.read()
+        self.assertEqual(memio.newlines, None)
+        memio.write("a\n")
+        force_decode()
+        self.assertEqual(memio.newlines, "\n")
+        memio.write("b\r\n")
+        force_decode()
+        self.assertEqual(memio.newlines, ("\n", "\r\n"))
+        memio.write("c\rd")
+        force_decode()
+        self.assertEqual(memio.newlines, ("\r", "\n", "\r\n"))
+
     def test_relative_seek(self):
         memio = self.ioclass()
 
@@ -388,29 +457,107 @@ class PyStringIOTest(MemoryTestMixin, unittest.TestCase):
         self.assertRaises(IOError, memio.seek, 1, 1)
         self.assertRaises(IOError, memio.seek, 1, 2)
 
-    # XXX: For the Python version of io.StringIO, this is highly
+    def test_textio_properties(self):
-    # dependent on the encoding used for the underlying buffer.
+        memio = self.ioclass()
-    # def test_widechar(self):
+
-    #     buf = self.buftype("\U0002030a\U00020347")
+        # These are just dummy values but we nevertheless check them for fear
-    #     memio = self.ioclass(buf)
+        # of unexpected breakage.
-    #
+        self.assertIsNone(memio.encoding)
-    #     self.assertEqual(memio.getvalue(), buf)
+        self.assertIsNone(memio.errors)
-    #     self.assertEqual(memio.write(buf), len(buf))
+        self.assertFalse(memio.line_buffering)
-    #     self.assertEqual(memio.tell(), len(buf))
+
-    #     self.assertEqual(memio.getvalue(), buf)
+    def test_newline_none(self):
-    #     self.assertEqual(memio.write(buf), len(buf))
+        # newline=None
-    #     self.assertEqual(memio.tell(), len(buf) * 2)
+        memio = self.ioclass("a\nb\r\nc\rd", newline=None)
-    #     self.assertEqual(memio.getvalue(), buf + buf)
+        self.assertEqual(list(memio), ["a\n", "b\n", "c\n", "d"])
+        memio.seek(0)
+        self.assertEqual(memio.read(1), "a")
+        self.assertEqual(memio.read(2), "\nb")
+        self.assertEqual(memio.read(2), "\nc")
+        self.assertEqual(memio.read(1), "\n")
+        memio = self.ioclass(newline=None)
+        self.assertEqual(2, memio.write("a\n"))
+        self.assertEqual(3, memio.write("b\r\n"))
+        self.assertEqual(3, memio.write("c\rd"))
+        memio.seek(0)
+        self.assertEqual(memio.read(), "a\nb\nc\nd")
+        memio = self.ioclass("a\r\nb", newline=None)
+        self.assertEqual(memio.read(3), "a\nb")
+
+    def test_newline_empty(self):
+        # newline=""
+        memio = self.ioclass("a\nb\r\nc\rd", newline="")
+        self.assertEqual(list(memio), ["a\n", "b\r\n", "c\r", "d"])
+        memio.seek(0)
+        self.assertEqual(memio.read(4), "a\nb\r")
+        self.assertEqual(memio.read(2), "\nc")
+        self.assertEqual(memio.read(1), "\r")
+        memio = self.ioclass(newline="")
+        self.assertEqual(2, memio.write("a\n"))
+        self.assertEqual(2, memio.write("b\r"))
+        self.assertEqual(2, memio.write("\nc"))
+        self.assertEqual(2, memio.write("\rd"))
+        memio.seek(0)
+        self.assertEqual(list(memio), ["a\n", "b\r\n", "c\r", "d"])
+
+    def test_newline_lf(self):
+        # newline="\n"
+        memio = self.ioclass("a\nb\r\nc\rd")
+        self.assertEqual(list(memio), ["a\n", "b\r\n", "c\rd"])
+
+    def test_newline_cr(self):
+        # newline="\r"
+        memio = self.ioclass("a\nb\r\nc\rd", newline="\r")
+        memio.seek(0)
+        self.assertEqual(memio.read(), "a\rb\r\rc\rd")
+        memio.seek(0)
+        self.assertEqual(list(memio), ["a\r", "b\r", "\r", "c\r", "d"])
+
+    def test_newline_crlf(self):
+        # newline="\r\n"
+        memio = self.ioclass("a\nb\r\nc\rd", newline="\r\n")
+        memio.seek(0)
+        self.assertEqual(memio.read(), "a\r\nb\r\r\nc\rd")
+        memio.seek(0)
+        self.assertEqual(list(memio), ["a\r\n", "b\r\r\n", "c\rd"])
+
+    def test_issue5265(self):
+        # StringIO can duplicate newlines in universal newlines mode
+        memio = self.ioclass("a\r\nb\r\n", newline=None)
+        self.assertEqual(memio.read(5), "a\nb\n")
+
 
-if has_c_implementation:
 class CBytesIOTest(PyBytesIOTest):
     ioclass = io.BytesIO
+    UnsupportedOperation = io.UnsupportedOperation
+
+    test_bytes_array = unittest.skip(
+        "array.array() does not have the new buffer API"
+    )(PyBytesIOTest.test_bytes_array)
+
+
+class CStringIOTest(PyStringIOTest):
+    ioclass = io.StringIO
+    UnsupportedOperation = io.UnsupportedOperation
+
+    # XXX: For the Python version of io.StringIO, this is highly
+    # dependent on the encoding used for the underlying buffer.
+    def test_widechar(self):
+        buf = self.buftype("\U0002030a\U00020347")
+        memio = self.ioclass(buf)
+
+        self.assertEqual(memio.getvalue(), buf)
+        self.assertEqual(memio.write(buf), len(buf))
+        self.assertEqual(memio.tell(), len(buf))
+        self.assertEqual(memio.getvalue(), buf)
+        self.assertEqual(memio.write(buf), len(buf))
+        self.assertEqual(memio.tell(), len(buf) * 2)
+        self.assertEqual(memio.getvalue(), buf + buf)
+
 
 def test_main():
-    tests = [PyBytesIOTest, PyStringIOTest]
+    tests = [PyBytesIOTest, PyStringIOTest, CBytesIOTest, CStringIOTest]
-    if has_c_implementation:
+    support.run_unittest(*tests)
-        tests.extend([CBytesIOTest])
-    test_support.run_unittest(*tests)
 
 if __name__ == '__main__':
     test_main()

Lib/test/test_support.py

@@ -29,7 +29,7 @@ __all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
            "run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest",
            "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
            "threading_cleanup", "reap_children", "cpython_only",
-           "check_impl_detail", "get_attribute"]
+           "check_impl_detail", "get_attribute", "py3k_bytes"]
 
 class Error(Exception):
     """Base class for regression test exceptions."""
@@ -968,3 +968,18 @@ def reap_children():
                     break
             except:
                 break
+
+def py3k_bytes(b):
+    """Emulate the py3k bytes() constructor.
+
+    NOTE: This is only a best effort function.
+    """
+    try:
+        # memoryview?
+        return b.tobytes()
+    except AttributeError:
+        try:
+            # iterable of ints?
+            return b"".join(chr(x) for x in b)
+        except TypeError:
+            return bytes(b)

Lib/test/test_univnewlines.py

@@ -1,20 +1,25 @@
 # Tests universal newline support for both reading and parsing files.
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import io
+import _pyio as pyio
 import unittest
 import os
 import sys
-from test import test_support
+from test import test_support as support
 
 if not hasattr(sys.stdin, 'newlines'):
-    raise unittest.SkipTest, \
+    raise unittest.SkipTest(
-        "This Python does not have universal newline support"
+        "This Python does not have universal newline support")
 
 FATX = 'x' * (2**14)
 
 DATA_TEMPLATE = [
     "line1=1",
-    "line2='this is a very long line designed to go past the magic " +
+    "line2='this is a very long line designed to go past any default " +
-        "hundred character limit that is inside fileobject.c and which " +
+        "buffer limits that exist in io.py but we also want to test " +
-        "is meant to speed up the common case, but we also want to test " +
         "the uncommon case, naturally.'",
     "def line3():pass",
     "line4 = '%s'" % FATX,
@@ -28,48 +33,50 @@ DATA_CRLF = "\r\n".join(DATA_TEMPLATE) + "\r\n"
 # before end-of-file.
 DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r"
 DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE]
-del x
 
 class TestGenericUnivNewlines(unittest.TestCase):
     # use a class variable DATA to define the data to write to the file
     # and a class variable NEWLINE to set the expected newlines value
-    READMODE = 'U'
+    READMODE = 'r'
     WRITEMODE = 'wb'
 
     def setUp(self):
-        with open(test_support.TESTFN, self.WRITEMODE) as fp:
+        data = self.DATA
-            fp.write(self.DATA)
+        if "b" in self.WRITEMODE:
+            data = data.encode("ascii")
+        with self.open(support.TESTFN, self.WRITEMODE) as fp:
+            fp.write(data)
 
     def tearDown(self):
         try:
-            os.unlink(test_support.TESTFN)
+            os.unlink(support.TESTFN)
         except:
             pass
 
     def test_read(self):
-        with open(test_support.TESTFN, self.READMODE) as fp:
+        with self.open(support.TESTFN, self.READMODE) as fp:
             data = fp.read()
         self.assertEqual(data, DATA_LF)
-        self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
+        self.assertEqual(set(fp.newlines), set(self.NEWLINE))
 
     def test_readlines(self):
-        with open(test_support.TESTFN, self.READMODE) as fp:
+        with self.open(support.TESTFN, self.READMODE) as fp:
             data = fp.readlines()
         self.assertEqual(data, DATA_SPLIT)
-        self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
+        self.assertEqual(set(fp.newlines), set(self.NEWLINE))
 
     def test_readline(self):
-        with open(test_support.TESTFN, self.READMODE) as fp:
+        with self.open(support.TESTFN, self.READMODE) as fp:
             data = []
             d = fp.readline()
             while d:
                 data.append(d)
                 d = fp.readline()
         self.assertEqual(data, DATA_SPLIT)
-        self.assertEqual(repr(fp.newlines), repr(self.NEWLINE))
+        self.assertEqual(set(fp.newlines), set(self.NEWLINE))
 
     def test_seek(self):
-        with open(test_support.TESTFN, self.READMODE) as fp:
+        with self.open(support.TESTFN, self.READMODE) as fp:
             fp.readline()
             pos = fp.tell()
             data = fp.readlines()
@@ -78,19 +85,6 @@ class TestGenericUnivNewlines(unittest.TestCase):
             data = fp.readlines()
         self.assertEqual(data, DATA_SPLIT[1:])
 
-    def test_execfile(self):
-        namespace = {}
-        execfile(test_support.TESTFN, namespace)
-        func = namespace['line3']
-        self.assertEqual(func.func_code.co_firstlineno, 3)
-        self.assertEqual(namespace['line4'], FATX)
-
-
-class TestNativeNewlines(TestGenericUnivNewlines):
-    NEWLINE = None
-    DATA = DATA_LF
-    READMODE = 'r'
-    WRITEMODE = 'w'
 
 class TestCRNewlines(TestGenericUnivNewlines):
     NEWLINE = '\r'
@@ -105,7 +99,7 @@ class TestCRLFNewlines(TestGenericUnivNewlines):
     DATA = DATA_CRLF
 
     def test_tell(self):
-        with open(test_support.TESTFN, self.READMODE) as fp:
+        with self.open(support.TESTFN, self.READMODE) as fp:
             self.assertEqual(repr(fp.newlines), repr(None))
             data = fp.readline()
             pos = fp.tell()
@@ -117,13 +111,22 @@ class TestMixedNewlines(TestGenericUnivNewlines):
 
 
 def test_main():
-    test_support.run_unittest(
+    base_tests = (TestCRNewlines,
-        TestNativeNewlines,
-        TestCRNewlines,
                   TestLFNewlines,
                   TestCRLFNewlines,
-        TestMixedNewlines
+                  TestMixedNewlines)
-     )
+    tests = []
+    # Test the C and Python implementations.
+    for test in base_tests:
+        class CTest(test):
+            open = io.open
+        CTest.__name__ = str("C" + test.__name__)
+        class PyTest(test):
+            open = staticmethod(pyio.open)
+        PyTest.__name__ = str("Py" + test.__name__)
+        tests.append(CTest)
+        tests.append(PyTest)
+    support.run_unittest(*tests)
 
 if __name__ == '__main__':
     test_main()

Misc/NEWS

@@ -317,6 +317,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #6215: All bug fixes and enhancements from the Python 3.1 io library
+  (including the fast C implementation) have been backported to the standard
+  ``io`` module.
+
 - Issue #6258: Support AMD64 in bdist_msi.
 
 - Issue #5262: Fixed bug in next rollover time computation in

Modules/_io/_iomodule.c

@@ -0,0 +1,735 @@
+/*
+    An implementation of the new I/O lib as defined by PEP 3116 - "New I/O"
+    
+    Classes defined here: UnsupportedOperation, BlockingIOError.
+    Functions defined here: open().
+    
+    Mostly written by Amaury Forgeot d'Arc
+*/
+
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include "structmember.h"
+#include "_iomodule.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif /* HAVE_SYS_TYPES_H */
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif /* HAVE_SYS_STAT_H */
+
+
+/* Various interned strings */
+
+PyObject *_PyIO_str_close;
+PyObject *_PyIO_str_closed;
+PyObject *_PyIO_str_decode;
+PyObject *_PyIO_str_encode;
+PyObject *_PyIO_str_fileno;
+PyObject *_PyIO_str_flush;
+PyObject *_PyIO_str_getstate;
+PyObject *_PyIO_str_isatty;
+PyObject *_PyIO_str_newlines;
+PyObject *_PyIO_str_nl;
+PyObject *_PyIO_str_read;
+PyObject *_PyIO_str_read1;
+PyObject *_PyIO_str_readable;
+PyObject *_PyIO_str_readinto;
+PyObject *_PyIO_str_readline;
+PyObject *_PyIO_str_reset;
+PyObject *_PyIO_str_seek;
+PyObject *_PyIO_str_seekable;
+PyObject *_PyIO_str_setstate;
+PyObject *_PyIO_str_tell;
+PyObject *_PyIO_str_truncate;
+PyObject *_PyIO_str_writable;
+PyObject *_PyIO_str_write;
+
+PyObject *_PyIO_empty_str;
+PyObject *_PyIO_empty_bytes;
+PyObject *_PyIO_zero;
+
+
+PyDoc_STRVAR(module_doc,
+"The io module provides the Python interfaces to stream handling. The\n"
+"builtin open function is defined in this module.\n"
+"\n"
+"At the top of the I/O hierarchy is the abstract base class IOBase. It\n"
+"defines the basic interface to a stream. Note, however, that there is no\n"
+"seperation between reading and writing to streams; implementations are\n"
+"allowed to throw an IOError if they do not support a given operation.\n"
+"\n"
+"Extending IOBase is RawIOBase which deals simply with the reading and\n"
+"writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide\n"
+"an interface to OS files.\n"
+"\n"
+"BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\n"
+"subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\n"
+"streams that are readable, writable, and both respectively.\n"
+"BufferedRandom provides a buffered interface to random access\n"
+"streams. BytesIO is a simple stream of in-memory bytes.\n"
+"\n"
+"Another IOBase subclass, TextIOBase, deals with the encoding and decoding\n"
+"of streams into text. TextIOWrapper, which extends it, is a buffered text\n"
+"interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\n"
+"is a in-memory stream for text.\n"
+"\n"
+"Argument names are not part of the specification, and only the arguments\n"
+"of open() are intended to be used as keyword arguments.\n"
+"\n"
+"data:\n"
+"\n"
+"DEFAULT_BUFFER_SIZE\n"
+"\n"
+"   An int containing the default buffer size used by the module's buffered\n"
+"   I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n"
+"   possible.\n"
+    );
+
+
+/*
+ * BlockingIOError extends IOError
+ */
+
+static int
+blockingioerror_init(PyBlockingIOErrorObject *self, PyObject *args,
+                     PyObject *kwds)
+{
+    PyObject *myerrno = NULL, *strerror = NULL;
+    PyObject *baseargs = NULL;
+    Py_ssize_t written = 0;
+
+    assert(PyTuple_Check(args));
+
+    self->written = 0;
+    if (!PyArg_ParseTuple(args, "OO|n:BlockingIOError",
+                          &myerrno, &strerror, &written))
+        return -1;
+
+    baseargs = PyTuple_Pack(2, myerrno, strerror);
+    if (baseargs == NULL)
+        return -1;
+    /* This will take care of initializing of myerrno and strerror members */
+    if (((PyTypeObject *)PyExc_IOError)->tp_init(
+                (PyObject *)self, baseargs, kwds) == -1) {
+        Py_DECREF(baseargs);
+        return -1;
+    }
+    Py_DECREF(baseargs);
+
+    self->written = written;
+    return 0;
+}
+
+static PyMemberDef blockingioerror_members[] = {
+    {"characters_written", T_PYSSIZET, offsetof(PyBlockingIOErrorObject, written), 0},
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject _PyExc_BlockingIOError = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "BlockingIOError", /*tp_name*/
+    sizeof(PyBlockingIOErrorObject), /*tp_basicsize*/
+    0,                          /*tp_itemsize*/
+    0,                          /*tp_dealloc*/
+    0,                          /*tp_print*/
+    0,                          /*tp_getattr*/
+    0,                          /*tp_setattr*/
+    0,                          /*tp_compare */
+    0,                          /*tp_repr*/
+    0,                          /*tp_as_number*/
+    0,                          /*tp_as_sequence*/
+    0,                          /*tp_as_mapping*/
+    0,                          /*tp_hash */
+    0,                          /*tp_call*/
+    0,                          /*tp_str*/
+    0,                          /*tp_getattro*/
+    0,                          /*tp_setattro*/
+    0,                          /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
+    PyDoc_STR("Exception raised when I/O would block "
+              "on a non-blocking I/O stream"), /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    0,                          /* tp_methods */
+    blockingioerror_members,    /* tp_members */
+    0,                          /* tp_getset */
+    0,                          /* tp_base */
+    0,                          /* tp_dict */
+    0,                          /* tp_descr_get */
+    0,                          /* tp_descr_set */
+    0,                          /* tp_dictoffset */
+    (initproc)blockingioerror_init, /* tp_init */
+    0,                          /* tp_alloc */
+    0,                          /* tp_new */
+};
+PyObject *PyExc_BlockingIOError = (PyObject *)&_PyExc_BlockingIOError;
+
+
+/*
+ * The main open() function
+ */
+PyDoc_STRVAR(open_doc,
+"Open file and return a stream.  Raise IOError upon failure.\n"
+"\n"
+"file is either a text or byte string giving the name (and the path\n"
+"if the file isn't in the current working directory) of the file to\n"
+"be opened or an integer file descriptor of the file to be\n"
+"wrapped. (If a file descriptor is given, it is closed when the\n"
+"returned I/O object is closed, unless closefd is set to False.)\n"
+"\n"
+"mode is an optional string that specifies the mode in which the file\n"
+"is opened. It defaults to 'r' which means open for reading in text\n"
+"mode.  Other common values are 'w' for writing (truncating the file if\n"
+"it already exists), and 'a' for appending (which on some Unix systems,\n"
+"means that all writes append to the end of the file regardless of the\n"
+"current seek position). In text mode, if encoding is not specified the\n"
+"encoding used is platform dependent. (For reading and writing raw\n"
+"bytes use binary mode and leave encoding unspecified.) The available\n"
+"modes are:\n"
+"\n"
+"========= ===============================================================\n"
+"Character Meaning\n"
+"--------- ---------------------------------------------------------------\n"
+"'r'       open for reading (default)\n"
+"'w'       open for writing, truncating the file first\n"
+"'a'       open for writing, appending to the end of the file if it exists\n"
+"'b'       binary mode\n"
+"'t'       text mode (default)\n"
+"'+'       open a disk file for updating (reading and writing)\n"
+"'U'       universal newline mode (for backwards compatibility; unneeded\n"
+"          for new code)\n"
+"========= ===============================================================\n"
+"\n"
+"The default mode is 'rt' (open for reading text). For binary random\n"
+"access, the mode 'w+b' opens and truncates the file to 0 bytes, while\n"
+"'r+b' opens the file without truncation.\n"
+"\n"
+"Python distinguishes between files opened in binary and text modes,\n"
+"even when the underlying operating system doesn't. Files opened in\n"
+"binary mode (appending 'b' to the mode argument) return contents as\n"
+"bytes objects without any decoding. In text mode (the default, or when\n"
+"'t' is appended to the mode argument), the contents of the file are\n"
+"returned as strings, the bytes having been first decoded using a\n"
+"platform-dependent encoding or using the specified encoding if given.\n"
+"\n"
+"buffering is an optional integer used to set the buffering policy. By\n"
+"default full buffering is on. Pass 0 to switch buffering off (only\n"
+"allowed in binary mode), 1 to set line buffering, and an integer > 1\n"
+"for full buffering.\n"
+"\n"
+"encoding is the name of the encoding used to decode or encode the\n"
+"file. This should only be used in text mode. The default encoding is\n"
+"platform dependent, but any encoding supported by Python can be\n"
+"passed.  See the codecs module for the list of supported encodings.\n"
+"\n"
+"errors is an optional string that specifies how encoding errors are to\n"
+"be handled---this argument should not be used in binary mode. Pass\n"
+"'strict' to raise a ValueError exception if there is an encoding error\n"
+"(the default of None has the same effect), or pass 'ignore' to ignore\n"
+"errors. (Note that ignoring encoding errors can lead to data loss.)\n"
+"See the documentation for codecs.register for a list of the permitted\n"
+"encoding error strings.\n"
+"\n"
+"newline controls how universal newlines works (it only applies to text\n"
+"mode). It can be None, '', '\\n', '\\r', and '\\r\\n'.  It works as\n"
+"follows:\n"
+"\n"
+"* On input, if newline is None, universal newlines mode is\n"
+"  enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n"
+"  these are translated into '\\n' before being returned to the\n"
+"  caller. If it is '', universal newline mode is enabled, but line\n"
+"  endings are returned to the caller untranslated. If it has any of\n"
+"  the other legal values, input lines are only terminated by the given\n"
+"  string, and the line ending is returned to the caller untranslated.\n"
+"\n"
+"* On output, if newline is None, any '\\n' characters written are\n"
+"  translated to the system default line separator, os.linesep. If\n"
+"  newline is '', no translation takes place. If newline is any of the\n"
+"  other legal values, any '\\n' characters written are translated to\n"
+"  the given string.\n"
+"\n"
+"If closefd is False, the underlying file descriptor will be kept open\n"
+"when the file is closed. This does not work when a file name is given\n"
+"and must be True in that case.\n"
+"\n"
+"open() returns a file object whose type depends on the mode, and\n"
+"through which the standard file operations such as reading and writing\n"
+"are performed. When open() is used to open a file in a text mode ('w',\n"
+"'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\n"
+"a file in a binary mode, the returned class varies: in read binary\n"
+"mode, it returns a BufferedReader; in write binary and append binary\n"
+"modes, it returns a BufferedWriter, and in read/write mode, it returns\n"
+"a BufferedRandom.\n"
+"\n"
+"It is also possible to use a string or bytearray as a file for both\n"
+"reading and writing. For strings StringIO can be used like a file\n"
+"opened in a text mode, and for bytes a BytesIO can be used like a file\n"
+"opened in a binary mode.\n"
+    );
+
+static PyObject *
+io_open(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    char *kwlist[] = {"file", "mode", "buffering",
+                      "encoding", "errors", "newline",
+                      "closefd", NULL};
+    PyObject *file;
+    char *mode = "r";
+    int buffering = -1, closefd = 1;
+    char *encoding = NULL, *errors = NULL, *newline = NULL;
+    unsigned i;
+
+    int reading = 0, writing = 0, appending = 0, updating = 0;
+    int text = 0, binary = 0, universal = 0;
+
+    char rawmode[5], *m;
+    int line_buffering, isatty;
+
+    PyObject *raw, *modeobj = NULL, *buffer = NULL, *wrapper = NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist,
+                                     &file, &mode, &buffering,
+                                     &encoding, &errors, &newline,
+                                     &closefd)) {
+        return NULL;
+    }
+
+    if (!PyUnicode_Check(file) &&
+	!PyBytes_Check(file) &&
+	!PyNumber_Check(file)) {
+        PyObject *repr = PyObject_Repr(file);
+        if (repr != NULL) {
+            PyErr_Format(PyExc_TypeError, "invalid file: %s",
+                         PyString_AS_STRING(repr));
+            Py_DECREF(repr);
+        }
+        return NULL;
+    }
+
+    /* Decode mode */
+    for (i = 0; i < strlen(mode); i++) {
+        char c = mode[i];
+
+        switch (c) {
+        case 'r':
+            reading = 1;
+            break;
+        case 'w':
+            writing = 1;
+            break;
+        case 'a':
+            appending = 1;
+            break;
+        case '+':
+            updating = 1;
+            break;
+        case 't':
+            text = 1;
+            break;
+        case 'b':
+            binary = 1;
+            break;
+        case 'U':
+            universal = 1;
+            reading = 1;
+            break;
+        default:
+            goto invalid_mode;
+        }
+
+        /* c must not be duplicated */
+        if (strchr(mode+i+1, c)) {
+          invalid_mode:
+            PyErr_Format(PyExc_ValueError, "invalid mode: '%s'", mode);
+            return NULL;
+        }
+
+    }
+
+    m = rawmode;
+    if (reading)   *(m++) = 'r';
+    if (writing)   *(m++) = 'w';
+    if (appending) *(m++) = 'a';
+    if (updating)  *(m++) = '+';
+    *m = '\0';
+
+    /* Parameters validation */
+    if (universal) {
+        if (writing || appending) {
+            PyErr_SetString(PyExc_ValueError,
+                            "can't use U and writing mode at once");
+            return NULL;
+        }
+        reading = 1;
+    }
+
+    if (text && binary) {
+        PyErr_SetString(PyExc_ValueError,
+                        "can't have text and binary mode at once");
+        return NULL;
+    }
+
+    if (reading + writing + appending > 1) {
+        PyErr_SetString(PyExc_ValueError,
+                        "must have exactly one of read/write/append mode");
+        return NULL;
+    }
+
+    if (binary && encoding != NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "binary mode doesn't take an encoding argument");
+        return NULL;
+    }
+
+    if (binary && errors != NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "binary mode doesn't take an errors argument");
+        return NULL;
+    }
+
+    if (binary && newline != NULL) {
+        PyErr_SetString(PyExc_ValueError,
+                        "binary mode doesn't take a newline argument");
+        return NULL;
+    }
+
+    /* Create the Raw file stream */
+    raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type,
+				"Osi", file, rawmode, closefd);
+    if (raw == NULL)
+        return NULL;
+
+    modeobj = PyUnicode_FromString(mode);
+    if (modeobj == NULL)
+        goto error;
+
+    /* buffering */
+    {
+        PyObject *res = PyObject_CallMethod(raw, "isatty", NULL);
+        if (res == NULL)
+            goto error;
+        isatty = PyLong_AsLong(res);
+        Py_DECREF(res);
+        if (isatty == -1 && PyErr_Occurred())
+            goto error;
+    }
+
+    if (buffering == 1 || (buffering < 0 && isatty)) {
+        buffering = -1;
+        line_buffering = 1;
+    }
+    else
+        line_buffering = 0;
+
+    if (buffering < 0) {
+        buffering = DEFAULT_BUFFER_SIZE;
+#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
+        {
+            struct stat st;
+            long fileno;
+            PyObject *res = PyObject_CallMethod(raw, "fileno", NULL);
+            if (res == NULL)
+                goto error;
+
+            fileno = PyInt_AsLong(res);
+            Py_DECREF(res);
+            if (fileno == -1 && PyErr_Occurred())
+                goto error;
+
+            if (fstat(fileno, &st) >= 0)
+                buffering = st.st_blksize;
+        }
+#endif
+    }
+    if (buffering < 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "invalid buffering size");
+        goto error;
+    }
+
+    /* if not buffering, returns the raw file object */
+    if (buffering == 0) {
+        if (!binary) {
+            PyErr_SetString(PyExc_ValueError,
+                            "can't have unbuffered text I/O");
+            goto error;
+        }
+
+        Py_DECREF(modeobj);
+        return raw;
+    }
+
+    /* wraps into a buffered file */
+    {
+        PyObject *Buffered_class;
+
+        if (updating)
+            Buffered_class = (PyObject *)&PyBufferedRandom_Type;
+        else if (writing || appending)
+            Buffered_class = (PyObject *)&PyBufferedWriter_Type;
+        else if (reading)
+            Buffered_class = (PyObject *)&PyBufferedReader_Type;
+        else {
+            PyErr_Format(PyExc_ValueError,
+                         "unknown mode: '%s'", mode);
+            goto error;
+        }
+
+        buffer = PyObject_CallFunction(Buffered_class, "Oi", raw, buffering);
+    }
+    Py_CLEAR(raw);
+    if (buffer == NULL)
+        goto error;
+
+
+    /* if binary, returns the buffered file */
+    if (binary) {
+        Py_DECREF(modeobj);
+        return buffer;
+    }
+
+    /* wraps into a TextIOWrapper */
+    wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type,
+				    "Osssi",
+				    buffer,
+				    encoding, errors, newline,
+				    line_buffering);
+    Py_CLEAR(buffer);
+    if (wrapper == NULL)
+        goto error;
+
+    if (PyObject_SetAttrString(wrapper, "mode", modeobj) < 0)
+        goto error;
+    Py_DECREF(modeobj);
+    return wrapper;
+
+  error:
+    Py_XDECREF(raw);
+    Py_XDECREF(modeobj);
+    Py_XDECREF(buffer);
+    Py_XDECREF(wrapper);
+    return NULL;
+}
+
+/*
+ * Private helpers for the io module.
+ */
+
+Py_off_t
+PyNumber_AsOff_t(PyObject *item, PyObject *err)
+{
+    Py_off_t result;
+    PyObject *runerr;
+    PyObject *value = PyNumber_Index(item);
+    if (value == NULL)
+        return -1;
+
+    if (PyInt_Check(value)) {
+        /* We assume a long always fits in a Py_off_t... */
+        result = (Py_off_t) PyInt_AS_LONG(value);
+        goto finish;
+    }
+
+    /* We're done if PyLong_AsSsize_t() returns without error. */
+    result = PyLong_AsOff_t(value);
+    if (result != -1 || !(runerr = PyErr_Occurred()))
+        goto finish;
+
+    /* Error handling code -- only manage OverflowError differently */
+    if (!PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError))
+        goto finish;
+
+    PyErr_Clear();
+    /* If no error-handling desired then the default clipping
+       is sufficient.
+     */
+    if (!err) {
+        assert(PyLong_Check(value));
+        /* Whether or not it is less than or equal to
+           zero is determined by the sign of ob_size
+        */
+        if (_PyLong_Sign(value) < 0)
+            result = PY_OFF_T_MIN;
+        else
+            result = PY_OFF_T_MAX;
+    }
+    else {
+        /* Otherwise replace the error with caller's error object. */
+        PyErr_Format(err,
+                     "cannot fit '%.200s' into an offset-sized integer",
+                     item->ob_type->tp_name);
+    }
+
+ finish:
+    Py_DECREF(value);
+    return result;
+}
+
+
+/*
+ * Module definition
+ */
+
+PyObject *_PyIO_os_module = NULL;
+PyObject *_PyIO_locale_module = NULL;
+PyObject *_PyIO_unsupported_operation = NULL;
+
+static PyMethodDef module_methods[] = {
+    {"open", (PyCFunction)io_open, METH_VARARGS|METH_KEYWORDS, open_doc},
+    {NULL, NULL}
+};
+
+PyMODINIT_FUNC
+init_io(void)
+{
+    PyObject *m = Py_InitModule4("_io", module_methods,
+                                 module_doc, NULL, PYTHON_API_VERSION);
+    if (m == NULL)
+        return;
+
+    /* put os in the module state */
+    _PyIO_os_module = PyImport_ImportModule("os");
+    if (_PyIO_os_module == NULL)
+        goto fail;
+
+#define ADD_TYPE(type, name) \
+    if (PyType_Ready(type) < 0) \
+        goto fail; \
+    Py_INCREF(type); \
+    if (PyModule_AddObject(m, name, (PyObject *)type) < 0) {  \
+        Py_DECREF(type); \
+        goto fail; \
+    }
+
+    /* DEFAULT_BUFFER_SIZE */
+    if (PyModule_AddIntMacro(m, DEFAULT_BUFFER_SIZE) < 0)
+        goto fail;
+
+    /* UnsupportedOperation inherits from ValueError and IOError */
+    _PyIO_unsupported_operation = PyObject_CallFunction(
+        (PyObject *)&PyType_Type, "s(OO){}",
+        "UnsupportedOperation", PyExc_ValueError, PyExc_IOError);
+    if (_PyIO_unsupported_operation == NULL)
+        goto fail;
+    Py_INCREF(_PyIO_unsupported_operation);
+    if (PyModule_AddObject(m, "UnsupportedOperation",
+                           _PyIO_unsupported_operation) < 0)
+        goto fail;
+
+    /* BlockingIOError */
+    _PyExc_BlockingIOError.tp_base = (PyTypeObject *) PyExc_IOError;
+    ADD_TYPE(&_PyExc_BlockingIOError, "BlockingIOError");
+
+    /* Concrete base types of the IO ABCs.
+       (the ABCs themselves are declared through inheritance in io.py)
+    */
+    ADD_TYPE(&PyIOBase_Type, "_IOBase");
+    ADD_TYPE(&PyRawIOBase_Type, "_RawIOBase");
+    ADD_TYPE(&PyBufferedIOBase_Type, "_BufferedIOBase");
+    ADD_TYPE(&PyTextIOBase_Type, "_TextIOBase");
+
+    /* Implementation of concrete IO objects. */
+    /* FileIO */
+    PyFileIO_Type.tp_base = &PyRawIOBase_Type;
+    ADD_TYPE(&PyFileIO_Type, "FileIO");
+
+    /* BytesIO */
+    PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type;
+    ADD_TYPE(&PyBytesIO_Type, "BytesIO");
+
+    /* StringIO */
+    PyStringIO_Type.tp_base = &PyTextIOBase_Type;
+    ADD_TYPE(&PyStringIO_Type, "StringIO");
+
+    /* BufferedReader */
+    PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type;
+    ADD_TYPE(&PyBufferedReader_Type, "BufferedReader");
+
+    /* BufferedWriter */
+    PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type;
+    ADD_TYPE(&PyBufferedWriter_Type, "BufferedWriter");
+
+    /* BufferedRWPair */
+    PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type;
+    ADD_TYPE(&PyBufferedRWPair_Type, "BufferedRWPair");
+
+    /* BufferedRandom */
+    PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type;
+    ADD_TYPE(&PyBufferedRandom_Type, "BufferedRandom");
+
+    /* TextIOWrapper */
+    PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type;
+    ADD_TYPE(&PyTextIOWrapper_Type, "TextIOWrapper");
+
+    /* IncrementalNewlineDecoder */
+    ADD_TYPE(&PyIncrementalNewlineDecoder_Type, "IncrementalNewlineDecoder");
+
+    /* Interned strings */
+    if (!(_PyIO_str_close = PyString_InternFromString("close")))
+        goto fail;
+    if (!(_PyIO_str_closed = PyString_InternFromString("closed")))
+        goto fail;
+    if (!(_PyIO_str_decode = PyString_InternFromString("decode")))
+        goto fail;
+    if (!(_PyIO_str_encode = PyString_InternFromString("encode")))
+        goto fail;
+    if (!(_PyIO_str_fileno = PyString_InternFromString("fileno")))
+        goto fail;
+    if (!(_PyIO_str_flush = PyString_InternFromString("flush")))
+        goto fail;
+    if (!(_PyIO_str_getstate = PyString_InternFromString("getstate")))
+        goto fail;
+    if (!(_PyIO_str_isatty = PyString_InternFromString("isatty")))
+        goto fail;
+    if (!(_PyIO_str_newlines = PyString_InternFromString("newlines")))
+        goto fail;
+    if (!(_PyIO_str_nl = PyString_InternFromString("\n")))
+        goto fail;
+    if (!(_PyIO_str_read = PyString_InternFromString("read")))
+        goto fail;
+    if (!(_PyIO_str_read1 = PyString_InternFromString("read1")))
+        goto fail;
+    if (!(_PyIO_str_readable = PyString_InternFromString("readable")))
+        goto fail;
+    if (!(_PyIO_str_readinto = PyString_InternFromString("readinto")))
+        goto fail;
+    if (!(_PyIO_str_readline = PyString_InternFromString("readline")))
+        goto fail;
+    if (!(_PyIO_str_reset = PyString_InternFromString("reset")))
+        goto fail;
+    if (!(_PyIO_str_seek = PyString_InternFromString("seek")))
+        goto fail;
+    if (!(_PyIO_str_seekable = PyString_InternFromString("seekable")))
+        goto fail;
+    if (!(_PyIO_str_setstate = PyString_InternFromString("setstate")))
+        goto fail;
+    if (!(_PyIO_str_tell = PyString_InternFromString("tell")))
+        goto fail;
+    if (!(_PyIO_str_truncate = PyString_InternFromString("truncate")))
+        goto fail;
+    if (!(_PyIO_str_write = PyString_InternFromString("write")))
+        goto fail;
+    if (!(_PyIO_str_writable = PyString_InternFromString("writable")))
+        goto fail;
+    
+    if (!(_PyIO_empty_str = PyUnicode_FromStringAndSize(NULL, 0)))
+        goto fail;
+    if (!(_PyIO_empty_bytes = PyBytes_FromStringAndSize(NULL, 0)))
+        goto fail;
+    if (!(_PyIO_zero = PyLong_FromLong(0L)))
+        goto fail;
+
+    return;
+
+  fail:
+    Py_CLEAR(_PyIO_os_module);
+    Py_CLEAR(_PyIO_unsupported_operation);
+    Py_DECREF(m);
+}

Modules/_io/_iomodule.h

@@ -0,0 +1,146 @@
+/*
+ * Declarations shared between the different parts of the io module
+ */
+
+/* ABCs */
+extern PyTypeObject PyIOBase_Type;
+extern PyTypeObject PyRawIOBase_Type;
+extern PyTypeObject PyBufferedIOBase_Type;
+extern PyTypeObject PyTextIOBase_Type;
+
+/* Concrete classes */
+extern PyTypeObject PyFileIO_Type;
+extern PyTypeObject PyBytesIO_Type;
+extern PyTypeObject PyStringIO_Type;
+extern PyTypeObject PyBufferedReader_Type;
+extern PyTypeObject PyBufferedWriter_Type;
+extern PyTypeObject PyBufferedRWPair_Type;
+extern PyTypeObject PyBufferedRandom_Type;
+extern PyTypeObject PyTextIOWrapper_Type;
+extern PyTypeObject PyIncrementalNewlineDecoder_Type;
+
+/* These functions are used as METH_NOARGS methods, are normally called
+ * with args=NULL, and return a new reference.
+ * BUT when args=Py_True is passed, they return a borrowed reference.
+ */
+extern PyObject* _PyIOBase_check_readable(PyObject *self, PyObject *args);
+extern PyObject* _PyIOBase_check_writable(PyObject *self, PyObject *args);
+extern PyObject* _PyIOBase_check_seekable(PyObject *self, PyObject *args);
+extern PyObject* _PyIOBase_check_closed(PyObject *self, PyObject *args);
+
+/* Helper for finalization.
+   This function will revive an object ready to be deallocated and try to
+   close() it. It returns 0 if the object can be destroyed, or -1 if it
+   is alive again. */
+extern int _PyIOBase_finalize(PyObject *self);
+
+/* Returns true if the given FileIO object is closed.
+   Doesn't check the argument type, so be careful! */
+extern int _PyFileIO_closed(PyObject *self);
+
+/* Shortcut to the core of the IncrementalNewlineDecoder.decode method */
+extern PyObject *_PyIncrementalNewlineDecoder_decode(
+    PyObject *self, PyObject *input, int final);
+
+/* Finds the first line ending between `start` and `end`.
+   If found, returns the index after the line ending and doesn't touch
+   `*consumed`.
+   If not found, returns -1 and sets `*consumed` to the number of characters
+   which can be safely put aside until another search.
+   
+   NOTE: for performance reasons, `end` must point to a NUL character ('\0'). 
+   Otherwise, the function will scan further and return garbage. */
+extern Py_ssize_t _PyIO_find_line_ending(
+    int translated, int universal, PyObject *readnl,
+    Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed);
+
+
+#define DEFAULT_BUFFER_SIZE (8 * 1024)  /* bytes */
+
+typedef struct {
+    /* This is the equivalent of PyException_HEAD in 3.x */
+    PyObject_HEAD
+    PyObject *dict;
+    PyObject *args;
+    PyObject *message;
+
+    PyObject *myerrno;
+    PyObject *strerror;
+    PyObject *filename; /* Not used, but part of the IOError object */
+    Py_ssize_t written;
+} PyBlockingIOErrorObject;
+PyAPI_DATA(PyObject *) PyExc_BlockingIOError;
+
+/*
+ * Offset type for positioning.
+ */
+
+#if defined(MS_WIN64) || defined(MS_WINDOWS)
+
+/* Windows uses long long for offsets */
+typedef PY_LONG_LONG Py_off_t;
+# define PyLong_AsOff_t     PyLong_AsLongLong
+# define PyLong_FromOff_t   PyLong_FromLongLong
+# define PY_OFF_T_MAX       PY_LLONG_MAX
+# define PY_OFF_T_MIN       PY_LLONG_MIN
+
+#else
+
+/* Other platforms use off_t */
+typedef off_t Py_off_t;
+#if (SIZEOF_OFF_T == SIZEOF_SIZE_T)
+# define PyLong_AsOff_t     PyLong_AsSsize_t
+# define PyLong_FromOff_t   PyLong_FromSsize_t
+# define PY_OFF_T_MAX       PY_SSIZE_T_MAX
+# define PY_OFF_T_MIN       PY_SSIZE_T_MIN
+#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG)
+# define PyLong_AsOff_t     PyLong_AsLongLong
+# define PyLong_FromOff_t   PyLong_FromLongLong
+# define PY_OFF_T_MAX       PY_LLONG_MAX
+# define PY_OFF_T_MIN       PY_LLONG_MIN
+#elif (SIZEOF_OFF_T == SIZEOF_LONG)
+# define PyLong_AsOff_t     PyLong_AsLong
+# define PyLong_FromOff_t   PyLong_FromLong
+# define PY_OFF_T_MAX       LONG_MAX
+# define PY_OFF_T_MIN       LONG_MIN
+#else
+# error off_t does not match either size_t, long, or long long!
+#endif
+
+#endif
+
+extern Py_off_t PyNumber_AsOff_t(PyObject *item, PyObject *err);
+
+/* Implementation details */
+
+extern PyObject *_PyIO_os_module;
+extern PyObject *_PyIO_locale_module;
+extern PyObject *_PyIO_unsupported_operation;
+
+extern PyObject *_PyIO_str_close;
+extern PyObject *_PyIO_str_closed;
+extern PyObject *_PyIO_str_decode;
+extern PyObject *_PyIO_str_encode;
+extern PyObject *_PyIO_str_fileno;
+extern PyObject *_PyIO_str_flush;
+extern PyObject *_PyIO_str_getstate;
+extern PyObject *_PyIO_str_isatty;
+extern PyObject *_PyIO_str_newlines;
+extern PyObject *_PyIO_str_nl;
+extern PyObject *_PyIO_str_read;
+extern PyObject *_PyIO_str_read1;
+extern PyObject *_PyIO_str_readable;
+extern PyObject *_PyIO_str_readinto;
+extern PyObject *_PyIO_str_readline;
+extern PyObject *_PyIO_str_reset;
+extern PyObject *_PyIO_str_seek;
+extern PyObject *_PyIO_str_seekable;
+extern PyObject *_PyIO_str_setstate;
+extern PyObject *_PyIO_str_tell;
+extern PyObject *_PyIO_str_truncate;
+extern PyObject *_PyIO_str_writable;
+extern PyObject *_PyIO_str_write;
+
+extern PyObject *_PyIO_empty_str;
+extern PyObject *_PyIO_empty_bytes;
+extern PyObject *_PyIO_zero;

Modules/_io/bytesio.c

@@ -1,4 +1,6 @@
 #include "Python.h"
+#include "structmember.h"       /* for offsetof() */
+#include "_iomodule.h"
 
 typedef struct {
     PyObject_HEAD
@@ -6,7 +8,9 @@ typedef struct {
     Py_ssize_t pos;
     Py_ssize_t string_size;
     size_t buf_size;
-} BytesIOObject;
+    PyObject *dict;
+    PyObject *weakreflist;
+} bytesio;
 
 #define CHECK_CLOSED(self)                                  \
     if ((self)->buf == NULL) {                              \
@@ -19,7 +23,7 @@ typedef struct {
    object. Returns the length between the current position to the
    next newline character. */
 static Py_ssize_t
-get_line(BytesIOObject *self, char **output)
+get_line(bytesio *self, char **output)
 {
     char *n;
     const char *str_end;
@@ -52,7 +56,7 @@ get_line(BytesIOObject *self, char **output)
    The caller should ensure that the 'size' argument is non-negative.  Returns
    0 on success, -1 otherwise. */
 static int
-resize_buffer(BytesIOObject *self, size_t size)
+resize_buffer(bytesio *self, size_t size)
 {
     /* Here, unsigned types are used to avoid dealing with signed integer
        overflow, which is undefined in C. */
@@ -104,7 +108,7 @@ resize_buffer(BytesIOObject *self, size_t size)
 /* Internal routine for writing a string of bytes to the buffer of a BytesIO
    object. Returns the number of bytes wrote, or -1 on error. */
 static Py_ssize_t
-write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len)
+write_bytes(bytesio *self, const char *bytes, Py_ssize_t len)
 {
     assert(self->buf != NULL);
     assert(self->pos >= 0);
@@ -142,17 +146,19 @@ write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len)
 }
 
 static PyObject *
-bytesio_get_closed(BytesIOObject *self)
+bytesio_get_closed(bytesio *self)
 {
-    if (self->buf == NULL)
+    if (self->buf == NULL) {
         Py_RETURN_TRUE;
-    else
+    }
+    else {
         Py_RETURN_FALSE;
     }
+}
 
 /* Generic getter for the writable, readable and seekable properties */
 static PyObject *
-return_true(BytesIOObject *self)
+return_true(bytesio *self)
 {
     Py_RETURN_TRUE;
 }
@@ -161,7 +167,7 @@ PyDoc_STRVAR(flush_doc,
 "flush() -> None.  Does nothing.");
 
 static PyObject *
-bytesio_flush(BytesIOObject *self)
+bytesio_flush(bytesio *self)
 {
     Py_RETURN_NONE;
 }
@@ -172,10 +178,10 @@ PyDoc_STRVAR(getval_doc,
 "Retrieve the entire contents of the BytesIO object.");
 
 static PyObject *
-bytesio_getvalue(BytesIOObject *self)
+bytesio_getvalue(bytesio *self)
 {
     CHECK_CLOSED(self);
-    return PyString_FromStringAndSize(self->buf, self->string_size);
+    return PyBytes_FromStringAndSize(self->buf, self->string_size);
 }
 
 PyDoc_STRVAR(isatty_doc,
@@ -185,7 +191,7 @@ PyDoc_STRVAR(isatty_doc,
 "to a tty-like device.");
 
 static PyObject *
-bytesio_isatty(BytesIOObject *self)
+bytesio_isatty(bytesio *self)
 {
     CHECK_CLOSED(self);
     Py_RETURN_FALSE;
@@ -195,10 +201,10 @@ PyDoc_STRVAR(tell_doc,
 "tell() -> current file position, an integer\n");
 
 static PyObject *
-bytesio_tell(BytesIOObject *self)
+bytesio_tell(bytesio *self)
 {
     CHECK_CLOSED(self);
-    return PyInt_FromSsize_t(self->pos);
+    return PyLong_FromSsize_t(self->pos);
 }
 
 PyDoc_STRVAR(read_doc,
@@ -208,7 +214,7 @@ PyDoc_STRVAR(read_doc,
 "Return an empty string at EOF.");
 
 static PyObject *
-bytesio_read(BytesIOObject *self, PyObject *args)
+bytesio_read(bytesio *self, PyObject *args)
 {
     Py_ssize_t size, n;
     char *output;
@@ -219,8 +225,8 @@ bytesio_read(BytesIOObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "|O:read", &arg))
         return NULL;
 
-    if (PyInt_Check(arg)) {
+    if (PyNumber_Check(arg)) {
-        size = PyInt_AsSsize_t(arg);
+        size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
         if (size == -1 && PyErr_Occurred())
             return NULL;
     }
@@ -246,7 +252,7 @@ bytesio_read(BytesIOObject *self, PyObject *args)
     output = self->buf + self->pos;
     self->pos += size;
 
-    return PyString_FromStringAndSize(output, size);
+    return PyBytes_FromStringAndSize(output, size);
 }
 
 
@@ -257,7 +263,7 @@ PyDoc_STRVAR(read1_doc,
 "Return an empty string at EOF.");
 
 static PyObject *
-bytesio_read1(BytesIOObject *self, PyObject *n)
+bytesio_read1(bytesio *self, PyObject *n)
 {
     PyObject *arg, *res;
 
@@ -277,7 +283,7 @@ PyDoc_STRVAR(readline_doc,
 "Return an empty string at EOF.\n");
 
 static PyObject *
-bytesio_readline(BytesIOObject *self, PyObject *args)
+bytesio_readline(bytesio *self, PyObject *args)
 {
     Py_ssize_t size, n;
     char *output;
@@ -288,8 +294,8 @@ bytesio_readline(BytesIOObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "|O:readline", &arg))
         return NULL;
 
-    if (PyInt_Check(arg)) {
+    if (PyNumber_Check(arg)) {
-        size = PyInt_AsSsize_t(arg);
+        size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
         if (size == -1 && PyErr_Occurred())
             return NULL;
     }
@@ -311,7 +317,7 @@ bytesio_readline(BytesIOObject *self, PyObject *args)
         self->pos -= size;
     }
 
-    return PyString_FromStringAndSize(output, n);
+    return PyBytes_FromStringAndSize(output, n);
 }
 
 PyDoc_STRVAR(readlines_doc,
@@ -322,7 +328,7 @@ PyDoc_STRVAR(readlines_doc,
 "total number of bytes in the lines returned.\n");
 
 static PyObject *
-bytesio_readlines(BytesIOObject *self, PyObject *args)
+bytesio_readlines(bytesio *self, PyObject *args)
 {
     Py_ssize_t maxsize, size, n;
     PyObject *result, *line;
@@ -334,8 +340,8 @@ bytesio_readlines(BytesIOObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "|O:readlines", &arg))
         return NULL;
 
-    if (PyInt_Check(arg)) {
+    if (PyNumber_Check(arg)) {
-        maxsize = PyInt_AsSsize_t(arg);
+        maxsize = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
         if (maxsize == -1 && PyErr_Occurred())
             return NULL;
     }
@@ -355,7 +361,7 @@ bytesio_readlines(BytesIOObject *self, PyObject *args)
         return NULL;
 
     while ((n = get_line(self, &output)) != 0) {
-        line = PyString_FromStringAndSize(output, n);
+        line = PyBytes_FromStringAndSize(output, n);
         if (!line)
             goto on_error;
         if (PyList_Append(result, line) == -1) {
@@ -381,25 +387,27 @@ PyDoc_STRVAR(readinto_doc,
 "is set not to block as has no data to read.");
 
 static PyObject *
-bytesio_readinto(BytesIOObject *self, PyObject *buffer)
+bytesio_readinto(bytesio *self, PyObject *args)
 {
-    void *raw_buffer;
+    Py_buffer buf;
     Py_ssize_t len;
 
     CHECK_CLOSED(self);
 
-    if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1)
+    if (!PyArg_ParseTuple(args, "w*", &buf))
         return NULL;
 
+    len = buf.len;
     if (self->pos + len > self->string_size)
         len = self->string_size - self->pos;
 
-    memcpy(raw_buffer, self->buf + self->pos, len);
+    memcpy(buf.buf, self->buf + self->pos, len);
     assert(self->pos + len < PY_SSIZE_T_MAX);
     assert(len >= 0);
     self->pos += len;
 
-    return PyInt_FromSsize_t(len);
+    PyBuffer_Release(&buf);
+    return PyLong_FromSsize_t(len);
 }
 
 PyDoc_STRVAR(truncate_doc,
@@ -409,7 +417,7 @@ PyDoc_STRVAR(truncate_doc,
 "Returns the new size.  Imply an absolute seek to the position size.");
 
 static PyObject *
-bytesio_truncate(BytesIOObject *self, PyObject *args)
+bytesio_truncate(bytesio *self, PyObject *args)
 {
     Py_ssize_t size;
     PyObject *arg = Py_None;
@@ -419,8 +427,8 @@ bytesio_truncate(BytesIOObject *self, PyObject *args)
     if (!PyArg_ParseTuple(args, "|O:truncate", &arg))
         return NULL;
 
-    if (PyInt_Check(arg)) {
+    if (PyNumber_Check(arg)) {
-        size = PyInt_AsSsize_t(arg);
+        size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
         if (size == -1 && PyErr_Occurred())
             return NULL;
     }
@@ -447,11 +455,11 @@ bytesio_truncate(BytesIOObject *self, PyObject *args)
     }
     self->pos = size;
 
-    return PyInt_FromSsize_t(size);
+    return PyLong_FromSsize_t(size);
 }
 
 static PyObject *
-bytesio_iternext(BytesIOObject *self)
+bytesio_iternext(bytesio *self)
 {
     char *next;
     Py_ssize_t n;
@@ -463,7 +471,7 @@ bytesio_iternext(BytesIOObject *self)
     if (!next || n == 0)
         return NULL;
 
-    return PyString_FromStringAndSize(next, n);
+    return PyBytes_FromStringAndSize(next, n);
 }
 
 PyDoc_STRVAR(seek_doc,
@@ -476,25 +484,19 @@ PyDoc_STRVAR(seek_doc,
 "Returns the new absolute position.");
 
 static PyObject *
-bytesio_seek(BytesIOObject *self, PyObject *args)
+bytesio_seek(bytesio *self, PyObject *args)
 {
-    PyObject *pos_obj, *mode_obj;
+    PyObject *posobj;
     Py_ssize_t pos;
     int mode = 0;
 
     CHECK_CLOSED(self);
 
-    /* Special-case for 2.x to prevent floats from passing through.
+    if (!PyArg_ParseTuple(args, "O|i:seek", &posobj, &mode))
-       This only needed to make a test in test_io succeed. */
-    if (!PyArg_UnpackTuple(args, "seek", 1, 2, &pos_obj, &mode_obj))
         return NULL;
-    if (PyFloat_Check(pos_obj)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "position argument must be an integer");
-        return NULL;
-    }
 
-    if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode))
+    pos = PyNumber_AsSsize_t(posobj, PyExc_OverflowError);
+    if (pos == -1 && PyErr_Occurred())
         return NULL;
     
     if (pos < 0 && mode == 0) {
@@ -532,7 +534,7 @@ bytesio_seek(BytesIOObject *self, PyObject *args)
         pos = 0;
     self->pos = pos;
 
-    return PyInt_FromSsize_t(self->pos);
+    return PyLong_FromSsize_t(self->pos);
 }
 
 PyDoc_STRVAR(write_doc,
@@ -541,31 +543,24 @@ PyDoc_STRVAR(write_doc,
 "Return the number of bytes written.");
 
 static PyObject *
-bytesio_write(BytesIOObject *self, PyObject *obj)
+bytesio_write(bytesio *self, PyObject *obj)
 {
-    const char *bytes;
-    Py_ssize_t size;
     Py_ssize_t n = 0;
+    Py_buffer buf;
+    PyObject *result = NULL;
 
     CHECK_CLOSED(self);
 
-    /* Special-case in 2.x to prevent unicode objects to pass through. */
+    if (PyObject_GetBuffer(obj, &buf, PyBUF_CONTIG_RO) < 0)
-    if (PyUnicode_Check(obj)) {
-	    PyErr_SetString(PyExc_TypeError,
-                        "expecting a bytes object, got unicode");
-        return NULL;
-    }
-
-    if (PyObject_AsReadBuffer(obj, (void *)&bytes, &size) < 0)
         return NULL;
 
-    if (size != 0) {
+    if (buf.len != 0)
-        n = write_bytes(self, bytes, size);
+        n = write_bytes(self, buf.buf, buf.len);
-        if (n < 0)
+    if (n >= 0)
-            return NULL;
+        result = PyLong_FromSsize_t(n);
-    }
 
-    return PyInt_FromSsize_t(n);
+    PyBuffer_Release(&buf);
+    return result;
 }
 
 PyDoc_STRVAR(writelines_doc,
@@ -576,7 +571,7 @@ PyDoc_STRVAR(writelines_doc,
 "each string.");
 
 static PyObject *
-bytesio_writelines(BytesIOObject *self, PyObject *v)
+bytesio_writelines(bytesio *self, PyObject *v)
 {
     PyObject *it, *item;
     PyObject *ret;
@@ -609,7 +604,7 @@ PyDoc_STRVAR(close_doc,
 "close() -> None.  Disable all I/O operations.");
 
 static PyObject *
-bytesio_close(BytesIOObject *self)
+bytesio_close(bytesio *self)
 {
     if (self->buf != NULL) {
         PyMem_Free(self->buf);
@@ -619,22 +614,23 @@ bytesio_close(BytesIOObject *self)
 }
 
 static void
-bytesio_dealloc(BytesIOObject *self)
+bytesio_dealloc(bytesio *self)
 {
     if (self->buf != NULL) {
         PyMem_Free(self->buf);
         self->buf = NULL;
     }
+    Py_TYPE(self)->tp_clear((PyObject *)self);
     Py_TYPE(self)->tp_free(self);
 }
 
 static PyObject *
 bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
-    BytesIOObject *self;
+    bytesio *self;
 
     assert(type != NULL && type->tp_alloc != NULL);
-    self = (BytesIOObject *)type->tp_alloc(type, 0);
+    self = (bytesio *)type->tp_alloc(type, 0);
     if (self == NULL)
         return NULL;
 
@@ -651,7 +647,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 }
 
 static int
-bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds)
+bytesio_init(bytesio *self, PyObject *args, PyObject *kwds)
 {
     PyObject *initvalue = NULL;
 
@@ -674,10 +670,28 @@ bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds)
     return 0;
 }
 
+static int
+bytesio_traverse(bytesio *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->dict);
+    Py_VISIT(self->weakreflist);
+    return 0;
+}
+
+static int
+bytesio_clear(bytesio *self)
+{
+    Py_CLEAR(self->dict);
+    if (self->weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *)self);
+    return 0;
+}
+
+
 static PyGetSetDef bytesio_getsetlist[] = {
     {"closed",  (getter)bytesio_get_closed, NULL,
      "True if the file is closed."},
-    {0},            /* sentinel */
+    {NULL},            /* sentinel */
 };
 
 static struct PyMethodDef bytesio_methods[] = {
@@ -691,7 +705,7 @@ static struct PyMethodDef bytesio_methods[] = {
     {"write",      (PyCFunction)bytesio_write,      METH_O, write_doc},
     {"writelines", (PyCFunction)bytesio_writelines, METH_O, writelines_doc},
     {"read1",      (PyCFunction)bytesio_read1,      METH_O, read1_doc},
-    {"readinto",   (PyCFunction)bytesio_readinto,   METH_O, readinto_doc},
+    {"readinto",   (PyCFunction)bytesio_readinto,   METH_VARARGS, readinto_doc},
     {"readline",   (PyCFunction)bytesio_readline,   METH_VARARGS, readline_doc},
     {"readlines",  (PyCFunction)bytesio_readlines,  METH_VARARGS, readlines_doc},
     {"read",       (PyCFunction)bytesio_read,       METH_VARARGS, read_doc},
@@ -707,16 +721,16 @@ PyDoc_STRVAR(bytesio_doc,
 "Create a buffered I/O implementation using an in-memory bytes\n"
 "buffer, ready for reading and writing.");
 
-static PyTypeObject BytesIO_Type = {
+PyTypeObject PyBytesIO_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "_bytesio._BytesIO",                       /*tp_name*/
+    "_io.BytesIO",                             /*tp_name*/
-    sizeof(BytesIOObject),                     /*tp_basicsize*/
+    sizeof(bytesio),                     /*tp_basicsize*/
     0,                                         /*tp_itemsize*/
     (destructor)bytesio_dealloc,               /*tp_dealloc*/
     0,                                         /*tp_print*/
     0,                                         /*tp_getattr*/
     0,                                         /*tp_setattr*/
-    0,                                         /*tp_compare*/
+    0,                                         /*tp_reserved*/
     0,                                         /*tp_repr*/
     0,                                         /*tp_as_number*/
     0,                                         /*tp_as_sequence*/
@@ -727,12 +741,13 @@ static PyTypeObject BytesIO_Type = {
     0,                                         /*tp_getattro*/
     0,                                         /*tp_setattro*/
     0,                                         /*tp_as_buffer*/
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  /*tp_flags*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
+    Py_TPFLAGS_HAVE_GC,                        /*tp_flags*/
     bytesio_doc,                               /*tp_doc*/
-    0,                                         /*tp_traverse*/
+    (traverseproc)bytesio_traverse,            /*tp_traverse*/
-    0,                                         /*tp_clear*/
+    (inquiry)bytesio_clear,                    /*tp_clear*/
     0,                                         /*tp_richcompare*/
-    0,                                         /*tp_weaklistoffset*/
+    offsetof(bytesio, weakreflist),      /*tp_weaklistoffset*/
     PyObject_SelfIter,                         /*tp_iter*/
     (iternextfunc)bytesio_iternext,            /*tp_iternext*/
     bytesio_methods,                           /*tp_methods*/
@@ -742,22 +757,8 @@ static PyTypeObject BytesIO_Type = {
     0,                                         /*tp_dict*/
     0,                                         /*tp_descr_get*/
     0,                                         /*tp_descr_set*/
-    0,                                         /*tp_dictoffset*/
+    offsetof(bytesio, dict),             /*tp_dictoffset*/
     (initproc)bytesio_init,                    /*tp_init*/
     0,                                         /*tp_alloc*/
     bytesio_new,                               /*tp_new*/
 };
-
-PyMODINIT_FUNC
-init_bytesio(void)
-{
-    PyObject *m;
-
-    if (PyType_Ready(&BytesIO_Type) < 0)
-        return;
-    m = Py_InitModule("_bytesio", NULL);
-    if (m == NULL)
-        return;
-    Py_INCREF(&BytesIO_Type);
-    PyModule_AddObject(m, "_BytesIO", (PyObject *)&BytesIO_Type);
-}

Modules/_io/fileio.c

@@ -6,6 +6,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <stddef.h> /* For offsetof */
+#include "_iomodule.h"
 
 /*
  * Known likely problems:
@@ -27,6 +28,20 @@
 #include <windows.h>
 #endif
 
+#if BUFSIZ < (8*1024)
+#define SMALLCHUNK (8*1024)
+#elif (BUFSIZ >= (2 << 25))
+#error "unreasonable BUFSIZ > 64MB defined"
+#else
+#define SMALLCHUNK BUFSIZ
+#endif
+
+#if SIZEOF_INT < 4
+#define BIGCHUNK  (512 * 32)
+#else
+#define BIGCHUNK  (512 * 1024)
+#endif
+
 typedef struct {
 	PyObject_HEAD
 	int fd;
@@ -35,55 +50,76 @@ typedef struct {
 	int seekable : 2; /* -1 means unknown */
 	int closefd : 1;
 	PyObject *weakreflist;
-} PyFileIOObject;
+	PyObject *dict;
+} fileio;
 
 PyTypeObject PyFileIO_Type;
 
 #define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type))
 
+int
+_PyFileIO_closed(PyObject *self)
+{
+	return ((fileio *)self)->fd < 0;
+}
+
 static PyObject *
 portable_lseek(int fd, PyObject *posobj, int whence);
 
-/* Returns 0 on success, errno (which is < 0) on failure. */
+static PyObject *portable_lseek(int fd, PyObject *posobj, int whence);
+
+/* Returns 0 on success, -1 with exception set on failure. */
 static int
-internal_close(PyFileIOObject *self)
+internal_close(fileio *self)
 {
+	int err = 0;
 	int save_errno = 0;
 	if (self->fd >= 0) {
 		int fd = self->fd;
 		self->fd = -1;
+		/* fd is accessible and someone else may have closed it */
+		if (_PyVerify_fd(fd)) {
 			Py_BEGIN_ALLOW_THREADS
-		if (close(fd) < 0)
+			err = close(fd);
+			if (err < 0)
 				save_errno = errno;
 			Py_END_ALLOW_THREADS
+		} else {
+			save_errno = errno;
+			err = -1;
 		}
-	return save_errno;
+	}
+	if (err < 0) {
+		errno = save_errno;
+		PyErr_SetFromErrno(PyExc_IOError);
+		return -1;
+	}
+	return 0;
 }
 
 static PyObject *
-fileio_close(PyFileIOObject *self)
+fileio_close(fileio *self)
 {
 	if (!self->closefd) {
 		self->fd = -1;
 		Py_RETURN_NONE;
 	}
 	errno = internal_close(self);
-	if (errno < 0) {
+	if (errno < 0)
-		PyErr_SetFromErrno(PyExc_IOError);
 		return NULL;
-	}
 
-	Py_RETURN_NONE;
+	return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type,
+				   "close", "O", self);
 }
 
 static PyObject *
-fileio_new(PyTypeObject *type, PyObject *args, PyObject *kews)
+fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
-	PyFileIOObject *self;
+	fileio *self;
 
 	assert(type != NULL && type->tp_alloc != NULL);
 
-	self = (PyFileIOObject *) type->tp_alloc(type, 0);
+	self = (fileio *) type->tp_alloc(type, 0);
 	if (self != NULL) {
 		self->fd = -1;
 		self->readable = 0;
@@ -101,7 +137,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kews)
    directories, so we need a check.  */
 
 static int
-dircheck(PyFileIOObject* self, char *name)
+dircheck(fileio* self, const char *name)
 {
 #if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR)
 	struct stat buf;
@@ -110,7 +146,8 @@ dircheck(PyFileIOObject* self, char *name)
 	if (fstat(self->fd, &buf) == 0 && S_ISDIR(buf.st_mode)) {
 		char *msg = strerror(EISDIR);
 		PyObject *exc;
-		internal_close(self);
+		if (internal_close(self))
+			return -1;
 
 		exc = PyObject_CallFunction(PyExc_IOError, "(iss)",
 					    EISDIR, msg, name);
@@ -144,9 +181,10 @@ check_fd(int fd)
 static int
 fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 {
-	PyFileIOObject *self = (PyFileIOObject *) oself;
+	fileio *self = (fileio *) oself;
 	static char *kwlist[] = {"file", "mode", "closefd", NULL};
-	char *name = NULL;
+	const char *name = NULL;
+	PyObject *nameobj, *stringobj = NULL;
 	char *mode = "r";
 	char *s;
 #ifdef MS_WINDOWS
@@ -165,42 +203,59 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 			return -1;
 	}
 
-	if (PyArg_ParseTupleAndKeywords(args, kwds, "i|si:fileio",
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio",
-					kwlist, &fd, &mode, &closefd)) {
+					 kwlist, &nameobj, &mode, &closefd))
+		return -1;
+
+	if (PyFloat_Check(nameobj)) {
+		PyErr_SetString(PyExc_TypeError,
+				"integer argument expected, got float");
+		return -1;
+	}
+
+	fd = PyLong_AsLong(nameobj);
 	if (fd < 0) {
+		if (!PyErr_Occurred()) {
 			PyErr_SetString(PyExc_ValueError,
 					"Negative filedescriptor");
 			return -1;
 		}
-		if (check_fd(fd))
-			return -1;
-	}
-	else {
 		PyErr_Clear();
+	}
 
 #ifdef MS_WINDOWS
 	if (GetVersion() < 0x80000000) {
 		/* On NT, so wide API available */
-		PyObject *po;
+		if (PyUnicode_Check(nameobj))
-		if (PyArg_ParseTupleAndKeywords(args, kwds, "U|si:fileio",
+			widename = PyUnicode_AS_UNICODE(nameobj);
-						kwlist, &po, &mode, &closefd)
-						) {
-			widename = PyUnicode_AS_UNICODE(po);
-		} else {
-			/* Drop the argument parsing error as narrow
-			   strings are also valid. */
-			PyErr_Clear();
-		}
 	}
 	if (widename == NULL)
 #endif
+	if (fd < 0)
 	{
-		if (!PyArg_ParseTupleAndKeywords(args, kwds, "et|si:fileio",
+		if (PyBytes_Check(nameobj) || PyByteArray_Check(nameobj)) {
-						 kwlist,
+			Py_ssize_t namelen;
-						 Py_FileSystemDefaultEncoding,
+			if (PyObject_AsCharBuffer(nameobj, &name, &namelen) < 0)
-						 &name, &mode, &closefd))
 				return -1;
 		}
+		else {
+			PyObject *u = PyUnicode_FromObject(nameobj);
+
+			if (u == NULL)
+				return -1;
+
+			stringobj = PyUnicode_AsEncodedString(
+				u, Py_FileSystemDefaultEncoding, "surrogateescape");
+			Py_DECREF(u);
+			if (stringobj == NULL)
+				return -1;
+			if (!PyBytes_Check(stringobj)) {
+				PyErr_SetString(PyExc_TypeError,
+						"encoder failed to return bytes");
+				goto error;
+			}
+			name = PyBytes_AS_STRING(stringobj);
+		}
 	}
 
 	s = mode;
@@ -266,6 +321,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 #endif
 
 	if (fd >= 0) {
+		if (check_fd(fd))
+			goto error;
 		self->fd = fd;
 		self->closefd = closefd;
 	}
@@ -299,6 +356,9 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 			goto error;
 	}
 
+	if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0)
+		goto error;
+
 	if (append) {
 		/* For consistent behaviour, we explicitly seek to the
 		   end of file (otherwise, it might be done only on the
@@ -315,24 +375,33 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 	ret = -1;
 
  done:
-	PyMem_Free(name);
+	Py_CLEAR(stringobj);
 	return ret;
 }
 
-static void
+static int
-fileio_dealloc(PyFileIOObject *self)
+fileio_traverse(fileio *self, visitproc visit, void *arg)
 {
+	Py_VISIT(self->dict);
+	return 0;
+}
+
+static int
+fileio_clear(fileio *self)
+{
+	Py_CLEAR(self->dict);
+	return 0;
+}
+
+static void
+fileio_dealloc(fileio *self)
+{
+	if (_PyIOBase_finalize((PyObject *) self) < 0)
+		return;
+	_PyObject_GC_UNTRACK(self);
 	if (self->weakreflist != NULL)
 		PyObject_ClearWeakRefs((PyObject *) self);
-
+	Py_CLEAR(self->dict);
-	if (self->fd >= 0 && self->closefd) {
-		errno = internal_close(self);
-		if (errno < 0) {
-			PySys_WriteStderr("close failed: [Errno %d] %s\n",
-                                          errno, strerror(errno));
-		}
-	}
-
 	Py_TYPE(self)->tp_free((PyObject *)self);
 }
 
@@ -351,7 +420,7 @@ err_mode(char *action)
 }
 
 static PyObject *
-fileio_fileno(PyFileIOObject *self)
+fileio_fileno(fileio *self)
 {
 	if (self->fd < 0)
 		return err_closed();
@@ -359,7 +428,7 @@ fileio_fileno(PyFileIOObject *self)
 }
 
 static PyObject *
-fileio_readable(PyFileIOObject *self)
+fileio_readable(fileio *self)
 {
 	if (self->fd < 0)
 		return err_closed();
@@ -367,7 +436,7 @@ fileio_readable(PyFileIOObject *self)
 }
 
 static PyObject *
-fileio_writable(PyFileIOObject *self)
+fileio_writable(fileio *self)
 {
 	if (self->fd < 0)
 		return err_closed();
@@ -375,25 +444,25 @@ fileio_writable(PyFileIOObject *self)
 }
 
 static PyObject *
-fileio_seekable(PyFileIOObject *self)
+fileio_seekable(fileio *self)
 {
 	if (self->fd < 0)
 		return err_closed();
 	if (self->seekable < 0) {
-		int ret;
+		PyObject *pos = portable_lseek(self->fd, NULL, SEEK_CUR);
-		Py_BEGIN_ALLOW_THREADS
+		if (pos == NULL) {
-		ret = lseek(self->fd, 0, SEEK_CUR);
+			PyErr_Clear();
-		Py_END_ALLOW_THREADS
-		if (ret < 0)
 			self->seekable = 0;
-		else
+		} else {
+			Py_DECREF(pos);
 			self->seekable = 1;
 		}
+	}
 	return PyBool_FromLong((long) self->seekable);
 }
 
 static PyObject *
-fileio_readinto(PyFileIOObject *self, PyObject *args)
+fileio_readinto(fileio *self, PyObject *args)
 {
 	Py_buffer pbuf;
 	Py_ssize_t n;
@@ -406,10 +475,13 @@ fileio_readinto(PyFileIOObject *self, PyObject *args)
 	if (!PyArg_ParseTuple(args, "w*", &pbuf))
 		return NULL;
 
+	if (_PyVerify_fd(self->fd)) {
 		Py_BEGIN_ALLOW_THREADS
 		errno = 0;
 		n = read(self->fd, pbuf.buf, pbuf.len);
 		Py_END_ALLOW_THREADS
+	} else
+		n = -1;
 	PyBuffer_Release(&pbuf);
 	if (n < 0) {
 		if (errno == EAGAIN)
@@ -421,23 +493,62 @@ fileio_readinto(PyFileIOObject *self, PyObject *args)
 	return PyLong_FromSsize_t(n);
 }
 
-#define DEFAULT_BUFFER_SIZE (8*1024)
+static size_t
+new_buffersize(fileio *self, size_t currentsize)
+{
+#ifdef HAVE_FSTAT
+	off_t pos, end;
+	struct stat st;
+	if (fstat(self->fd, &st) == 0) {
+		end = st.st_size;
+		pos = lseek(self->fd, 0L, SEEK_CUR);
+		/* Files claiming a size smaller than SMALLCHUNK may
+		   actually be streaming pseudo-files. In this case, we
+		   apply the more aggressive algorithm below.
+		*/
+		if (end >= SMALLCHUNK && end >= pos && pos >= 0) {
+			/* Add 1 so if the file were to grow we'd notice. */
+			return currentsize + end - pos + 1;
+		}
+	}
+#endif
+	if (currentsize > SMALLCHUNK) {
+		/* Keep doubling until we reach BIGCHUNK;
+		   then keep adding BIGCHUNK. */
+		if (currentsize <= BIGCHUNK)
+			return currentsize + currentsize;
+		else
+			return currentsize + BIGCHUNK;
+	}
+	return currentsize + SMALLCHUNK;
+}
 
 static PyObject *
-fileio_readall(PyFileIOObject *self)
+fileio_readall(fileio *self)
 {
 	PyObject *result;
 	Py_ssize_t total = 0;
 	int n;
 
-	result = PyString_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE);
+	if (!_PyVerify_fd(self->fd))
+		return PyErr_SetFromErrno(PyExc_IOError);
+
+	result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK);
 	if (result == NULL)
 		return NULL;
 
 	while (1) {
-		Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE;
+		size_t newsize = new_buffersize(self, total);
-		if (PyString_GET_SIZE(result) < newsize) {
+		if (newsize > PY_SSIZE_T_MAX || newsize <= 0) {
-			if (_PyString_Resize(&result, newsize) < 0) {
+			PyErr_SetString(PyExc_OverflowError,
+				"unbounded read returned more bytes "
+				"than a Python string can hold ");
+			Py_DECREF(result);
+			return NULL;
+		}
+
+		if (PyBytes_GET_SIZE(result) < (Py_ssize_t)newsize) {
+			if (_PyBytes_Resize(&result, newsize) < 0) {
 				if (total == 0) {
 					Py_DECREF(result);
 					return NULL;
@@ -449,7 +560,7 @@ fileio_readall(PyFileIOObject *self)
 		Py_BEGIN_ALLOW_THREADS
 		errno = 0;
 		n = read(self->fd,
-			 PyString_AS_STRING(result) + total,
+			 PyBytes_AS_STRING(result) + total,
 			 newsize - total);
 		Py_END_ALLOW_THREADS
 		if (n == 0)
@@ -468,8 +579,8 @@ fileio_readall(PyFileIOObject *self)
 		total += n;
 	}
 
-	if (PyString_GET_SIZE(result) > total) {
+	if (PyBytes_GET_SIZE(result) > total) {
-		if (_PyString_Resize(&result, total) < 0) {
+		if (_PyBytes_Resize(&result, total) < 0) {
 			/* This should never happen, but just in case */
 			Py_DECREF(result);
 			return NULL;
@@ -479,7 +590,7 @@ fileio_readall(PyFileIOObject *self)
 }
 
 static PyObject *
-fileio_read(PyFileIOObject *self, PyObject *args)
+fileio_read(fileio *self, PyObject *args)
 {
 	char *ptr;
 	Py_ssize_t n;
@@ -498,17 +609,21 @@ fileio_read(PyFileIOObject *self, PyObject *args)
 		return fileio_readall(self);
 	}
 
-	bytes = PyString_FromStringAndSize(NULL, size);
+	bytes = PyBytes_FromStringAndSize(NULL, size);
 	if (bytes == NULL)
 		return NULL;
-	ptr = PyString_AS_STRING(bytes);
+	ptr = PyBytes_AS_STRING(bytes);
 
+	if (_PyVerify_fd(self->fd)) {
 		Py_BEGIN_ALLOW_THREADS
 		errno = 0;
 		n = read(self->fd, ptr, size);
 		Py_END_ALLOW_THREADS
+	} else
+		n = -1;
 
 	if (n < 0) {
+		Py_DECREF(bytes);
 		if (errno == EAGAIN)
 			Py_RETURN_NONE;
 		PyErr_SetFromErrno(PyExc_IOError);
@@ -516,7 +631,7 @@ fileio_read(PyFileIOObject *self, PyObject *args)
 	}
 
 	if (n != size) {
-		if (_PyString_Resize(&bytes, n) < 0) {
+		if (_PyBytes_Resize(&bytes, n) < 0) {
 			Py_DECREF(bytes);
 			return NULL;
 		}
@@ -526,7 +641,7 @@ fileio_read(PyFileIOObject *self, PyObject *args)
 }
 
 static PyObject *
-fileio_write(PyFileIOObject *self, PyObject *args)
+fileio_write(fileio *self, PyObject *args)
 {
 	Py_buffer pbuf;
 	Py_ssize_t n;
@@ -539,10 +654,13 @@ fileio_write(PyFileIOObject *self, PyObject *args)
 	if (!PyArg_ParseTuple(args, "s*", &pbuf))
 		return NULL;
 
+	if (_PyVerify_fd(self->fd)) {
 		Py_BEGIN_ALLOW_THREADS
 		errno = 0;
 		n = write(self->fd, pbuf.buf, pbuf.len);
 		Py_END_ALLOW_THREADS
+	} else
+		n = -1;
 
 	PyBuffer_Release(&pbuf);
 
@@ -558,12 +676,6 @@ fileio_write(PyFileIOObject *self, PyObject *args)
 
 /* XXX Windows support below is likely incomplete */
 
-#if defined(MS_WIN64) || defined(MS_WINDOWS)
-typedef PY_LONG_LONG Py_off_t;
-#else
-typedef off_t Py_off_t;
-#endif
-
 /* Cribbed from posix_lseek() */
 static PyObject *
 portable_lseek(int fd, PyObject *posobj, int whence)
@@ -601,6 +713,7 @@ portable_lseek(int fd, PyObject *posobj, int whence)
 			return NULL;
 	}
 
+	if (_PyVerify_fd(fd)) {
 		Py_BEGIN_ALLOW_THREADS
 #if defined(MS_WIN64) || defined(MS_WINDOWS)
 		res = _lseeki64(fd, pos, whence);
@@ -608,6 +721,8 @@ portable_lseek(int fd, PyObject *posobj, int whence)
 		res = lseek(fd, pos, whence);
 #endif
 		Py_END_ALLOW_THREADS
+	} else
+		res = -1;
 	if (res < 0)
 		return PyErr_SetFromErrno(PyExc_IOError);
 
@@ -619,7 +734,7 @@ portable_lseek(int fd, PyObject *posobj, int whence)
 }
 
 static PyObject *
-fileio_seek(PyFileIOObject *self, PyObject *args)
+fileio_seek(fileio *self, PyObject *args)
 {
 	PyObject *posobj;
 	int whence = 0;
@@ -634,7 +749,7 @@ fileio_seek(PyFileIOObject *self, PyObject *args)
 }
 
 static PyObject *
-fileio_tell(PyFileIOObject *self, PyObject *args)
+fileio_tell(fileio *self, PyObject *args)
 {
 	if (self->fd < 0)
 		return err_closed();
@@ -644,7 +759,7 @@ fileio_tell(PyFileIOObject *self, PyObject *args)
 
 #ifdef HAVE_FTRUNCATE
 static PyObject *
-fileio_truncate(PyFileIOObject *self, PyObject *args)
+fileio_truncate(fileio *self, PyObject *args)
 {
 	PyObject *posobj = NULL;
 	Py_off_t pos;
@@ -670,13 +785,15 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
 		/* Move to the position to be truncated. */
                 posobj = portable_lseek(fd, posobj, 0);
         }
+	if (posobj == NULL)
+		return NULL;
 
 #if defined(HAVE_LARGEFILE_SUPPORT)
 	pos = PyLong_AsLongLong(posobj);
 #else
 	pos = PyLong_AsLong(posobj);
 #endif
-	if (PyErr_Occurred())
+	if (pos == -1 && PyErr_Occurred())
 		return NULL;
 
 #ifdef MS_WINDOWS
@@ -714,7 +831,7 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
 #endif
 
 static char *
-mode_string(PyFileIOObject *self)
+mode_string(fileio *self)
 {
 	if (self->readable) {
 		if (self->writable)
@@ -727,17 +844,37 @@ mode_string(PyFileIOObject *self)
 }
 
 static PyObject *
-fileio_repr(PyFileIOObject *self)
+fileio_repr(fileio *self)
 {
-        if (self->fd < 0)
+	PyObject *nameobj, *res;
-		return PyString_FromFormat("_fileio._FileIO(-1)");
 
-	return PyString_FromFormat("_fileio._FileIO(%d, '%s')",
+        if (self->fd < 0)
+		return PyString_FromFormat("<_io.FileIO [closed]>");
+
+	nameobj = PyObject_GetAttrString((PyObject *) self, "name");
+	if (nameobj == NULL) {
+		if (PyErr_ExceptionMatches(PyExc_AttributeError))
+			PyErr_Clear();
+		else
+			return NULL;
+		res = PyString_FromFormat("<_io.FileIO fd=%d mode='%s'>",
 					   self->fd, mode_string(self));
 	}
+	else {
+		PyObject *repr = PyObject_Repr(nameobj);
+		Py_DECREF(nameobj);
+		if (repr == NULL)
+			return NULL;
+		res = PyString_FromFormat("<_io.FileIO name=%s mode='%s'>",
+					   PyString_AS_STRING(repr),
+					   mode_string(self));
+		Py_DECREF(repr);
+	}
+	return res;
+}
 
 static PyObject *
-fileio_isatty(PyFileIOObject *self)
+fileio_isatty(fileio *self)
 {
 	long res;
 
@@ -806,7 +943,7 @@ PyDoc_STRVAR(tell_doc,
 "tell() -> int.	 Current file position");
 
 PyDoc_STRVAR(readinto_doc,
-"readinto() -> Undocumented.  Don't use this; it may go away.");
+"readinto() -> Same as RawIOBase.readinto().");
 
 PyDoc_STRVAR(close_doc,
 "close() -> None.  Close the file.\n"
@@ -848,21 +985,21 @@ static PyMethodDef fileio_methods[] = {
 /* 'closed' and 'mode' are attributes for backwards compatibility reasons. */
 
 static PyObject *
-get_closed(PyFileIOObject *self, void *closure)
+get_closed(fileio *self, void *closure)
 {
 	return PyBool_FromLong((long)(self->fd < 0));
 }
 
 static PyObject *
-get_closefd(PyFileIOObject *self, void *closure)
+get_closefd(fileio *self, void *closure)
 {
 	return PyBool_FromLong((long)(self->closefd));
 }
 
 static PyObject *
-get_mode(PyFileIOObject *self, void *closure)
+get_mode(fileio *self, void *closure)
 {
-	return PyString_FromString(mode_string(self));
+	return PyUnicode_FromString(mode_string(self));
 }
 
 static PyGetSetDef fileio_getsetlist[] = {
@@ -870,19 +1007,19 @@ static PyGetSetDef fileio_getsetlist[] = {
 	{"closefd", (getter)get_closefd, NULL, 
 		"True if the file descriptor will be closed"},
 	{"mode", (getter)get_mode, NULL, "String giving the file mode"},
-	{0},
+	{NULL},
 };
 
 PyTypeObject PyFileIO_Type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
-	"_FileIO",
+	"_io.FileIO",
-	sizeof(PyFileIOObject),
+	sizeof(fileio),
 	0,
 	(destructor)fileio_dealloc,		/* tp_dealloc */
 	0,					/* tp_print */
 	0,					/* tp_getattr */
 	0,					/* tp_setattr */
-	0,					/* tp_compare */
+	0,					/* tp_reserved */
 	(reprfunc)fileio_repr,			/* tp_repr */
 	0,					/* tp_as_number */
 	0,					/* tp_as_sequence */
@@ -893,12 +1030,13 @@ PyTypeObject PyFileIO_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
+			| Py_TPFLAGS_HAVE_GC,	/* tp_flags */
 	fileio_doc,				/* tp_doc */
-	0,					/* tp_traverse */
+	(traverseproc)fileio_traverse,		/* tp_traverse */
-	0,					/* tp_clear */
+	(inquiry)fileio_clear,			/* tp_clear */
 	0,					/* tp_richcompare */
-	offsetof(PyFileIOObject, weakreflist),	/* tp_weaklistoffset */
+	offsetof(fileio, weakreflist),	/* tp_weaklistoffset */
 	0,					/* tp_iter */
 	0,					/* tp_iternext */
 	fileio_methods,				/* tp_methods */
@@ -908,28 +1046,9 @@ PyTypeObject PyFileIO_Type = {
 	0,					/* tp_dict */
 	0,					/* tp_descr_get */
 	0,					/* tp_descr_set */
-	0,					/* tp_dictoffset */
+	offsetof(fileio, dict),         /* tp_dictoffset */
 	fileio_init,				/* tp_init */
 	PyType_GenericAlloc,			/* tp_alloc */
 	fileio_new,				/* tp_new */
-	PyObject_Del,				/* tp_free */
+	PyObject_GC_Del,			/* tp_free */
 };
-
-static PyMethodDef module_methods[] = {
-	{NULL, NULL}
-};
-
-PyMODINIT_FUNC
-init_fileio(void)
-{
-	PyObject *m;	/* a module object */
-
-	m = Py_InitModule3("_fileio", module_methods,
-			   "Fast implementation of io.FileIO.");
-	if (m == NULL)
-		return;
-	if (PyType_Ready(&PyFileIO_Type) < 0)
-		return;
-	Py_INCREF(&PyFileIO_Type);
-	PyModule_AddObject(m, "_FileIO", (PyObject *) &PyFileIO_Type);
-}

Modules/_io/iobase.c

@@ -0,0 +1,894 @@
+/*
+    An implementation of the I/O abstract base classes hierarchy
+    as defined by PEP 3116 - "New I/O"
+    
+    Classes defined here: IOBase, RawIOBase.
+    
+    Written by Amaury Forgeot d'Arc and Antoine Pitrou
+*/
+
+
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include "structmember.h"
+#include "_iomodule.h"
+
+/*
+ * IOBase class, an abstract class
+ */
+
+typedef struct {
+    PyObject_HEAD
+    
+    PyObject *dict;
+    PyObject *weakreflist;
+} iobase;
+
+PyDoc_STRVAR(iobase_doc,
+    "The abstract base class for all I/O classes, acting on streams of\n"
+    "bytes. There is no public constructor.\n"
+    "\n"
+    "This class provides dummy implementations for many methods that\n"
+    "derived classes can override selectively; the default implementations\n"
+    "represent a file that cannot be read, written or seeked.\n"
+    "\n"
+    "Even though IOBase does not declare read, readinto, or write because\n"
+    "their signatures will vary, implementations and clients should\n"
+    "consider those methods part of the interface. Also, implementations\n"
+    "may raise a IOError when operations they do not support are called.\n"
+    "\n"
+    "The basic type used for binary data read from or written to a file is\n"
+    "bytes. bytearrays are accepted too, and in some cases (such as\n"
+    "readinto) needed. Text I/O classes work with str data.\n"
+    "\n"
+    "Note that calling any method (even inquiries) on a closed stream is\n"
+    "undefined. Implementations may raise IOError in this case.\n"
+    "\n"
+    "IOBase (and its subclasses) support the iterator protocol, meaning\n"
+    "that an IOBase object can be iterated over yielding the lines in a\n"
+    "stream.\n"
+    "\n"
+    "IOBase also supports the :keyword:`with` statement. In this example,\n"
+    "fp is closed after the suite of the with statment is complete:\n"
+    "\n"
+    "with open('spam.txt', 'r') as fp:\n"
+    "    fp.write('Spam and eggs!')\n");
+
+/* Use this macro whenever you want to check the internal `closed` status
+   of the IOBase object rather than the virtual `closed` attribute as returned
+   by whatever subclass. */
+
+#define IS_CLOSED(self) \
+    PyObject_HasAttrString(self, "__IOBase_closed")
+
+/* Internal methods */
+static PyObject *
+iobase_unsupported(const char *message)
+{
+    PyErr_SetString(_PyIO_unsupported_operation, message);
+    return NULL;
+}
+
+/* Positionning */
+
+PyDoc_STRVAR(iobase_seek_doc,
+    "Change stream position.\n"
+    "\n"
+    "Change the stream position to byte offset offset. offset is\n"
+    "interpreted relative to the position indicated by whence.  Values\n"
+    "for whence are:\n"
+    "\n"
+    "* 0 -- start of stream (the default); offset should be zero or positive\n"
+    "* 1 -- current stream position; offset may be negative\n"
+    "* 2 -- end of stream; offset is usually negative\n"
+    "\n"
+    "Return the new absolute position.");
+
+static PyObject *
+iobase_seek(PyObject *self, PyObject *args)
+{
+    return iobase_unsupported("seek");
+}
+
+PyDoc_STRVAR(iobase_tell_doc,
+             "Return current stream position.");
+
+static PyObject *
+iobase_tell(PyObject *self, PyObject *args)
+{
+    return PyObject_CallMethod(self, "seek", "ii", 0, 1);
+}
+
+PyDoc_STRVAR(iobase_truncate_doc,
+    "Truncate file to size bytes.\n"
+    "\n"
+    "Size defaults to the current IO position as reported by tell().  Return\n"
+    "the new size.");
+
+static PyObject *
+iobase_truncate(PyObject *self, PyObject *args)
+{
+    return iobase_unsupported("truncate");
+}
+
+/* Flush and close methods */
+
+PyDoc_STRVAR(iobase_flush_doc,
+    "Flush write buffers, if applicable.\n"
+    "\n"
+    "This is not implemented for read-only and non-blocking streams.\n");
+
+static PyObject *
+iobase_flush(PyObject *self, PyObject *args)
+{
+    /* XXX Should this return the number of bytes written??? */
+    if (IS_CLOSED(self)) {
+        PyErr_SetString(PyExc_ValueError, "I/O operation on closed file.");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(iobase_close_doc,
+    "Flush and close the IO object.\n"
+    "\n"
+    "This method has no effect if the file is already closed.\n");
+
+static int
+iobase_closed(PyObject *self)
+{
+    PyObject *res;
+    int closed;
+    /* This gets the derived attribute, which is *not* __IOBase_closed
+       in most cases! */
+    res = PyObject_GetAttr(self, _PyIO_str_closed);
+    if (res == NULL)
+        return 0;
+    closed = PyObject_IsTrue(res);
+    Py_DECREF(res);
+    return closed;
+}
+
+static PyObject *
+iobase_closed_get(PyObject *self, void *context)
+{
+    return PyBool_FromLong(IS_CLOSED(self));
+}
+
+PyObject *
+_PyIOBase_check_closed(PyObject *self, PyObject *args)
+{
+    if (iobase_closed(self)) {
+        PyErr_SetString(PyExc_ValueError, "I/O operation on closed file.");
+        return NULL;
+    }
+    if (args == Py_True)
+        return Py_None;
+    else
+        Py_RETURN_NONE;
+}
+
+/* XXX: IOBase thinks it has to maintain its own internal state in
+   `__IOBase_closed` and call flush() by itself, but it is redundant with
+   whatever behaviour a non-trivial derived class will implement. */
+
+static PyObject *
+iobase_close(PyObject *self, PyObject *args)
+{
+    PyObject *res;
+
+    if (IS_CLOSED(self))
+        Py_RETURN_NONE;
+
+    res = PyObject_CallMethodObjArgs(self, _PyIO_str_flush, NULL);
+    PyObject_SetAttrString(self, "__IOBase_closed", Py_True);
+    if (res == NULL) {
+        /* If flush() fails, just give up */
+        if (PyErr_ExceptionMatches(PyExc_IOError))
+            PyErr_Clear();
+        else
+            return NULL;
+    }
+    Py_XDECREF(res);
+    Py_RETURN_NONE;
+}
+
+/* Finalization and garbage collection support */
+
+int
+_PyIOBase_finalize(PyObject *self)
+{
+    PyObject *res;
+    PyObject *tp, *v, *tb;
+    int closed = 1;
+    int is_zombie;
+
+    /* If _PyIOBase_finalize() is called from a destructor, we need to
+       resurrect the object as calling close() can invoke arbitrary code. */
+    is_zombie = (Py_REFCNT(self) == 0);
+    if (is_zombie) {
+        ++Py_REFCNT(self);
+    }
+    PyErr_Fetch(&tp, &v, &tb);
+    /* If `closed` doesn't exist or can't be evaluated as bool, then the
+       object is probably in an unusable state, so ignore. */
+    res = PyObject_GetAttr(self, _PyIO_str_closed);
+    if (res == NULL)
+        PyErr_Clear();
+    else {
+        closed = PyObject_IsTrue(res);
+        Py_DECREF(res);
+        if (closed == -1)
+            PyErr_Clear();
+    }
+    if (closed == 0) {
+        res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_close,
+                                          NULL);
+        /* Silencing I/O errors is bad, but printing spurious tracebacks is
+           equally as bad, and potentially more frequent (because of
+           shutdown issues). */
+        if (res == NULL)
+            PyErr_Clear();
+        else
+            Py_DECREF(res);
+    }
+    PyErr_Restore(tp, v, tb);
+    if (is_zombie) {
+        if (--Py_REFCNT(self) != 0) {
+            /* The object lives again. The following code is taken from
+               slot_tp_del in typeobject.c. */
+            Py_ssize_t refcnt = Py_REFCNT(self);
+            _Py_NewReference(self);
+            Py_REFCNT(self) = refcnt;
+            /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+             * we need to undo that. */
+            _Py_DEC_REFTOTAL;
+            /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+             * chain, so no more to do there.
+             * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+             * _Py_NewReference bumped tp_allocs:  both of those need to be
+             * undone.
+             */
+#ifdef COUNT_ALLOCS
+            --Py_TYPE(self)->tp_frees;
+            --Py_TYPE(self)->tp_allocs;
+#endif
+            return -1;
+        }
+    }
+    return 0;
+}
+
+static int
+iobase_traverse(iobase *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->dict);
+    return 0;
+}
+
+static int
+iobase_clear(iobase *self)
+{
+    if (_PyIOBase_finalize((PyObject *) self) < 0)
+        return -1;
+    Py_CLEAR(self->dict);
+    return 0;
+}
+
+/* Destructor */
+
+static void
+iobase_dealloc(iobase *self)
+{
+    /* NOTE: since IOBaseObject has its own dict, Python-defined attributes
+       are still available here for close() to use.
+       However, if the derived class declares a __slots__, those slots are
+       already gone.
+    */
+    if (_PyIOBase_finalize((PyObject *) self) < 0) {
+        /* When called from a heap type's dealloc, the type will be
+           decref'ed on return (see e.g. subtype_dealloc in typeobject.c). */
+        if (PyType_HasFeature(Py_TYPE(self), Py_TPFLAGS_HEAPTYPE))
+            Py_INCREF(Py_TYPE(self));
+        return;
+    }
+    _PyObject_GC_UNTRACK(self);
+    if (self->weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) self);
+    Py_CLEAR(self->dict);
+    Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+/* Inquiry methods */
+
+PyDoc_STRVAR(iobase_seekable_doc,
+    "Return whether object supports random access.\n"
+    "\n"
+    "If False, seek(), tell() and truncate() will raise IOError.\n"
+    "This method may need to do a test seek().");
+
+static PyObject *
+iobase_seekable(PyObject *self, PyObject *args)
+{
+    Py_RETURN_FALSE;
+}
+
+PyObject *
+_PyIOBase_check_seekable(PyObject *self, PyObject *args)
+{
+    PyObject *res  = PyObject_CallMethodObjArgs(self, _PyIO_str_seekable, NULL);
+    if (res == NULL)
+        return NULL;
+    if (res != Py_True) {
+        Py_CLEAR(res);
+        PyErr_SetString(PyExc_IOError, "File or stream is not seekable.");
+        return NULL;
+    }
+    if (args == Py_True) {
+        Py_DECREF(res);
+    }
+    return res;
+}
+
+PyDoc_STRVAR(iobase_readable_doc,
+    "Return whether object was opened for reading.\n"
+    "\n"
+    "If False, read() will raise IOError.");
+
+static PyObject *
+iobase_readable(PyObject *self, PyObject *args)
+{
+    Py_RETURN_FALSE;
+}
+
+/* May be called with any object */
+PyObject *
+_PyIOBase_check_readable(PyObject *self, PyObject *args)
+{
+    PyObject *res  = PyObject_CallMethodObjArgs(self, _PyIO_str_readable, NULL);
+    if (res == NULL)
+        return NULL;
+    if (res != Py_True) {
+        Py_CLEAR(res);
+        PyErr_SetString(PyExc_IOError, "File or stream is not readable.");
+        return NULL;
+    }
+    if (args == Py_True) {
+        Py_DECREF(res);
+    }
+    return res;
+}
+
+PyDoc_STRVAR(iobase_writable_doc,
+    "Return whether object was opened for writing.\n"
+    "\n"
+    "If False, read() will raise IOError.");
+
+static PyObject *
+iobase_writable(PyObject *self, PyObject *args)
+{
+    Py_RETURN_FALSE;
+}
+
+/* May be called with any object */
+PyObject *
+_PyIOBase_check_writable(PyObject *self, PyObject *args)
+{
+    PyObject *res  = PyObject_CallMethodObjArgs(self, _PyIO_str_writable, NULL);
+    if (res == NULL)
+        return NULL;
+    if (res != Py_True) {
+        Py_CLEAR(res);
+        PyErr_SetString(PyExc_IOError, "File or stream is not writable.");
+        return NULL;
+    }
+    if (args == Py_True) {
+        Py_DECREF(res);
+    }
+    return res;
+}
+
+/* Context manager */
+
+static PyObject *
+iobase_enter(PyObject *self, PyObject *args)
+{
+    if (_PyIOBase_check_closed(self, Py_True) == NULL)
+        return NULL;
+
+    Py_INCREF(self);
+    return self;
+}
+
+static PyObject *
+iobase_exit(PyObject *self, PyObject *args)
+{
+    return PyObject_CallMethodObjArgs(self, _PyIO_str_close, NULL);
+}
+
+/* Lower-level APIs */
+
+/* XXX Should these be present even if unimplemented? */
+
+PyDoc_STRVAR(iobase_fileno_doc,
+    "Returns underlying file descriptor if one exists.\n"
+    "\n"
+    "An IOError is raised if the IO object does not use a file descriptor.\n");
+
+static PyObject *
+iobase_fileno(PyObject *self, PyObject *args)
+{
+    return iobase_unsupported("fileno");
+}
+
+PyDoc_STRVAR(iobase_isatty_doc,
+    "Return whether this is an 'interactive' stream.\n"
+    "\n"
+    "Return False if it can't be determined.\n");
+
+static PyObject *
+iobase_isatty(PyObject *self, PyObject *args)
+{
+    if (_PyIOBase_check_closed(self, Py_True) == NULL)
+        return NULL;
+    Py_RETURN_FALSE;
+}
+
+/* Readline(s) and writelines */
+
+PyDoc_STRVAR(iobase_readline_doc,
+    "Read and return a line from the stream.\n"
+    "\n"
+    "If limit is specified, at most limit bytes will be read.\n"
+    "\n"
+    "The line terminator is always b'\n' for binary files; for text\n"
+    "files, the newlines argument to open can be used to select the line\n"
+    "terminator(s) recognized.\n");
+
+static PyObject *
+iobase_readline(PyObject *self, PyObject *args)
+{
+    /* For backwards compatibility, a (slowish) readline(). */
+
+    Py_ssize_t limit = -1;
+    int has_peek = 0;
+    PyObject *buffer, *result;
+    Py_ssize_t old_size = -1;
+
+    if (!PyArg_ParseTuple(args, "|n:readline", &limit)) {
+        return NULL;
+    }
+
+    if (PyObject_HasAttrString(self, "peek"))
+        has_peek = 1;
+
+    buffer = PyByteArray_FromStringAndSize(NULL, 0);
+    if (buffer == NULL)
+        return NULL;
+
+    while (limit < 0 || Py_SIZE(buffer) < limit) {
+        Py_ssize_t nreadahead = 1;
+        PyObject *b;
+
+        if (has_peek) {
+            PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1);
+            if (readahead == NULL)
+                goto fail;
+            if (!PyBytes_Check(readahead)) {
+                PyErr_Format(PyExc_IOError,
+                             "peek() should have returned a bytes object, "
+                             "not '%.200s'", Py_TYPE(readahead)->tp_name);
+                Py_DECREF(readahead);
+                goto fail;
+            }
+            if (PyBytes_GET_SIZE(readahead) > 0) {
+                Py_ssize_t n = 0;
+                const char *buf = PyBytes_AS_STRING(readahead);
+                if (limit >= 0) {
+                    do {
+                        if (n >= PyBytes_GET_SIZE(readahead) || n >= limit)
+                            break;
+                        if (buf[n++] == '\n')
+                            break;
+                    } while (1);
+                }
+                else {
+                    do {
+                        if (n >= PyBytes_GET_SIZE(readahead))
+                            break;
+                        if (buf[n++] == '\n')
+                            break;
+                    } while (1);
+                }
+                nreadahead = n;
+            }
+            Py_DECREF(readahead);
+        }
+
+        b = PyObject_CallMethod(self, "read", "n", nreadahead);
+        if (b == NULL)
+            goto fail;
+        if (!PyBytes_Check(b)) {
+            PyErr_Format(PyExc_IOError,
+                         "read() should have returned a bytes object, "
+                         "not '%.200s'", Py_TYPE(b)->tp_name);
+            Py_DECREF(b);
+            goto fail;
+        }
+        if (PyBytes_GET_SIZE(b) == 0) {
+            Py_DECREF(b);
+            break;
+        }
+
+        old_size = PyByteArray_GET_SIZE(buffer);
+        PyByteArray_Resize(buffer, old_size + PyBytes_GET_SIZE(b));
+        memcpy(PyByteArray_AS_STRING(buffer) + old_size,
+               PyBytes_AS_STRING(b), PyBytes_GET_SIZE(b));
+
+        Py_DECREF(b);
+
+        if (PyByteArray_AS_STRING(buffer)[PyByteArray_GET_SIZE(buffer) - 1] == '\n')
+            break;
+    }
+
+    result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(buffer),
+                                       PyByteArray_GET_SIZE(buffer));
+    Py_DECREF(buffer);
+    return result;
+  fail:
+    Py_DECREF(buffer);
+    return NULL;
+}
+
+static PyObject *
+iobase_iter(PyObject *self)
+{
+    if (_PyIOBase_check_closed(self, Py_True) == NULL)
+        return NULL;
+
+    Py_INCREF(self);
+    return self;
+}
+
+static PyObject *
+iobase_iternext(PyObject *self)
+{
+    PyObject *line = PyObject_CallMethodObjArgs(self, _PyIO_str_readline, NULL);
+
+    if (line == NULL)
+        return NULL;
+
+    if (PyObject_Size(line) == 0) {
+        Py_DECREF(line);
+        return NULL;
+    }
+
+    return line;
+}
+
+PyDoc_STRVAR(iobase_readlines_doc,
+    "Return a list of lines from the stream.\n"
+    "\n"
+    "hint can be specified to control the number of lines read: no more\n"
+    "lines will be read if the total size (in bytes/characters) of all\n"
+    "lines so far exceeds hint.");
+
+static PyObject *
+iobase_readlines(PyObject *self, PyObject *args)
+{
+    Py_ssize_t hint = -1, length = 0;
+    PyObject *hintobj = Py_None, *result;
+
+    if (!PyArg_ParseTuple(args, "|O:readlines", &hintobj)) {
+        return NULL;
+    }
+    if (hintobj != Py_None) {
+        hint = PyNumber_AsSsize_t(hintobj, PyExc_ValueError);
+        if (hint == -1 && PyErr_Occurred())
+            return NULL;
+    }
+
+    result = PyList_New(0);
+    if (result == NULL)
+        return NULL;
+
+    if (hint <= 0) {
+        /* XXX special-casing this made sense in the Python version in order
+           to remove the bytecode interpretation overhead, but it could
+           probably be removed here. */
+        PyObject *ret = PyObject_CallMethod(result, "extend", "O", self);
+        if (ret == NULL) {
+            Py_DECREF(result);
+            return NULL;
+        }
+        Py_DECREF(ret);
+        return result;
+    }
+
+    while (1) {
+        PyObject *line = PyIter_Next(self);
+        if (line == NULL) {
+            if (PyErr_Occurred()) {
+                Py_DECREF(result);
+                return NULL;
+            }
+            else
+                break; /* StopIteration raised */
+        }
+
+        if (PyList_Append(result, line) < 0) {
+            Py_DECREF(line);
+            Py_DECREF(result);
+            return NULL;
+        }
+        length += PyObject_Size(line);
+        Py_DECREF(line);
+
+        if (length > hint)
+            break;
+    }
+    return result;
+}
+
+static PyObject *
+iobase_writelines(PyObject *self, PyObject *args)
+{
+    PyObject *lines, *iter, *res;
+
+    if (!PyArg_ParseTuple(args, "O:writelines", &lines)) {
+        return NULL;
+    }
+
+    if (_PyIOBase_check_closed(self, Py_True) == NULL)
+        return NULL;
+
+    iter = PyObject_GetIter(lines);
+    if (iter == NULL)
+        return NULL;
+
+    while (1) {
+        PyObject *line = PyIter_Next(iter);
+        if (line == NULL) {
+            if (PyErr_Occurred()) {
+                Py_DECREF(iter);
+                return NULL;
+            }
+            else
+                break; /* Stop Iteration */
+        }
+
+        res = PyObject_CallMethodObjArgs(self, _PyIO_str_write, line, NULL);
+        Py_DECREF(line);
+        if (res == NULL) {
+            Py_DECREF(iter);
+            return NULL;
+        }
+        Py_DECREF(res);
+    }
+    Py_DECREF(iter);
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef iobase_methods[] = {
+    {"seek", iobase_seek, METH_VARARGS, iobase_seek_doc},
+    {"tell", iobase_tell, METH_NOARGS, iobase_tell_doc},
+    {"truncate", iobase_truncate, METH_VARARGS, iobase_truncate_doc},
+    {"flush", iobase_flush, METH_NOARGS, iobase_flush_doc},
+    {"close", iobase_close, METH_NOARGS, iobase_close_doc},
+
+    {"seekable", iobase_seekable, METH_NOARGS, iobase_seekable_doc},
+    {"readable", iobase_readable, METH_NOARGS, iobase_readable_doc},
+    {"writable", iobase_writable, METH_NOARGS, iobase_writable_doc},
+
+    {"_checkClosed",   _PyIOBase_check_closed, METH_NOARGS},
+    {"_checkSeekable", _PyIOBase_check_seekable, METH_NOARGS},
+    {"_checkReadable", _PyIOBase_check_readable, METH_NOARGS},
+    {"_checkWritable", _PyIOBase_check_writable, METH_NOARGS},
+
+    {"fileno", iobase_fileno, METH_NOARGS, iobase_fileno_doc},
+    {"isatty", iobase_isatty, METH_NOARGS, iobase_isatty_doc},
+
+    {"__enter__", iobase_enter, METH_NOARGS},
+    {"__exit__", iobase_exit, METH_VARARGS},
+
+    {"readline", iobase_readline, METH_VARARGS, iobase_readline_doc},
+    {"readlines", iobase_readlines, METH_VARARGS, iobase_readlines_doc},
+    {"writelines", iobase_writelines, METH_VARARGS},
+
+    {NULL, NULL}
+};
+
+static PyGetSetDef iobase_getset[] = {
+    {"closed", (getter)iobase_closed_get, NULL, NULL},
+    {NULL}
+};
+
+
+PyTypeObject PyIOBase_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_io._IOBase",              /*tp_name*/
+    sizeof(iobase),             /*tp_basicsize*/
+    0,                          /*tp_itemsize*/
+    (destructor)iobase_dealloc, /*tp_dealloc*/
+    0,                          /*tp_print*/
+    0,                          /*tp_getattr*/
+    0,                          /*tp_setattr*/
+    0,                          /*tp_compare */
+    0,                          /*tp_repr*/
+    0,                          /*tp_as_number*/
+    0,                          /*tp_as_sequence*/
+    0,                          /*tp_as_mapping*/
+    0,                          /*tp_hash */
+    0,                          /*tp_call*/
+    0,                          /*tp_str*/
+    0,                          /*tp_getattro*/
+    0,                          /*tp_setattro*/
+    0,                          /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
+        | Py_TPFLAGS_HAVE_GC,   /*tp_flags*/
+    iobase_doc,                 /* tp_doc */
+    (traverseproc)iobase_traverse, /* tp_traverse */
+    (inquiry)iobase_clear,      /* tp_clear */
+    0,                          /* tp_richcompare */
+    offsetof(iobase, weakreflist), /* tp_weaklistoffset */
+    iobase_iter,                /* tp_iter */
+    iobase_iternext,            /* tp_iternext */
+    iobase_methods,             /* tp_methods */
+    0,                          /* tp_members */
+    iobase_getset,              /* tp_getset */
+    0,                          /* tp_base */
+    0,                          /* tp_dict */
+    0,                          /* tp_descr_get */
+    0,                          /* tp_descr_set */
+    offsetof(iobase, dict),     /* tp_dictoffset */
+    0,                          /* tp_init */
+    0,                          /* tp_alloc */
+    PyType_GenericNew,          /* tp_new */
+};
+
+
+/*
+ * RawIOBase class, Inherits from IOBase.
+ */
+PyDoc_STRVAR(rawiobase_doc,
+             "Base class for raw binary I/O.");
+
+/*
+ * The read() method is implemented by calling readinto(); derived classes
+ * that want to support read() only need to implement readinto() as a
+ * primitive operation.  In general, readinto() can be more efficient than
+ * read().
+ *
+ * (It would be tempting to also provide an implementation of readinto() in
+ * terms of read(), in case the latter is a more suitable primitive operation,
+ * but that would lead to nasty recursion in case a subclass doesn't implement
+ * either.)
+*/
+
+static PyObject *
+rawiobase_read(PyObject *self, PyObject *args)
+{
+    Py_ssize_t n = -1;
+    PyObject *b, *res;
+
+    if (!PyArg_ParseTuple(args, "|n:read", &n)) {
+        return NULL;
+    }
+
+    if (n < 0)
+        return PyObject_CallMethod(self, "readall", NULL);
+
+    /* TODO: allocate a bytes object directly instead and manually construct
+       a writable memoryview pointing to it. */
+    b = PyByteArray_FromStringAndSize(NULL, n);
+    if (b == NULL)
+        return NULL;
+
+    res = PyObject_CallMethodObjArgs(self, _PyIO_str_readinto, b, NULL);
+    if (res == NULL) {
+        Py_DECREF(b);
+        return NULL;
+    }
+
+    n = PyNumber_AsSsize_t(res, PyExc_ValueError);
+    Py_DECREF(res);
+    if (n == -1 && PyErr_Occurred()) {
+        Py_DECREF(b);
+        return NULL;
+    }
+
+    res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n);
+    Py_DECREF(b);
+    return res;
+}
+
+
+PyDoc_STRVAR(rawiobase_readall_doc,
+             "Read until EOF, using multiple read() call.");
+
+static PyObject *
+rawiobase_readall(PyObject *self, PyObject *args)
+{
+    int r;
+    PyObject *chunks = PyList_New(0);
+    PyObject *result;
+    
+    if (chunks == NULL)
+        return NULL;
+
+    while (1) {
+        PyObject *data = PyObject_CallMethod(self, "read",
+                                             "i", DEFAULT_BUFFER_SIZE);
+        if (!data) {
+            Py_DECREF(chunks);
+            return NULL;
+        }
+        if (!PyBytes_Check(data)) {
+            Py_DECREF(chunks);
+            Py_DECREF(data);
+            PyErr_SetString(PyExc_TypeError, "read() should return bytes");
+            return NULL;
+        }
+        if (PyBytes_GET_SIZE(data) == 0) {
+            /* EOF */
+            Py_DECREF(data);
+            break;
+        }
+        r = PyList_Append(chunks, data);
+        Py_DECREF(data);
+        if (r < 0) {
+            Py_DECREF(chunks);
+            return NULL;
+        }
+    }
+    result = _PyBytes_Join(_PyIO_empty_bytes, chunks);
+    Py_DECREF(chunks);
+    return result;
+}
+
+static PyMethodDef rawiobase_methods[] = {
+    {"read", rawiobase_read, METH_VARARGS},
+    {"readall", rawiobase_readall, METH_NOARGS, rawiobase_readall_doc},
+    {NULL, NULL}
+};
+
+PyTypeObject PyRawIOBase_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_io._RawIOBase",                /*tp_name*/
+    0,                          /*tp_basicsize*/
+    0,                          /*tp_itemsize*/
+    0,                          /*tp_dealloc*/
+    0,                          /*tp_print*/
+    0,                          /*tp_getattr*/
+    0,                          /*tp_setattr*/
+    0,                          /*tp_compare */
+    0,                          /*tp_repr*/
+    0,                          /*tp_as_number*/
+    0,                          /*tp_as_sequence*/
+    0,                          /*tp_as_mapping*/
+    0,                          /*tp_hash */
+    0,                          /*tp_call*/
+    0,                          /*tp_str*/
+    0,                          /*tp_getattro*/
+    0,                          /*tp_setattro*/
+    0,                          /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  /*tp_flags*/
+    rawiobase_doc,              /* tp_doc */
+    0,                          /* tp_traverse */
+    0,                          /* tp_clear */
+    0,                          /* tp_richcompare */
+    0,                          /* tp_weaklistoffset */
+    0,                          /* tp_iter */
+    0,                          /* tp_iternext */
+    rawiobase_methods,          /* tp_methods */
+    0,                          /* tp_members */
+    0,                          /* tp_getset */
+    &PyIOBase_Type,             /* tp_base */
+    0,                          /* tp_dict */
+    0,                          /* tp_descr_get */
+    0,                          /* tp_descr_set */
+    0,                          /* tp_dictoffset */
+    0,                          /* tp_init */
+    0,                          /* tp_alloc */
+    0,                          /* tp_new */
+};

Modules/_io/stringio.c

@@ -0,0 +1,756 @@
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include "structmember.h"
+#include "_iomodule.h"
+
+/* Implementation note: the buffer is always at least one character longer
+   than the enclosed string, for proper functioning of _PyIO_find_line_ending.
+*/
+
+typedef struct {
+    PyObject_HEAD
+    Py_UNICODE *buf;
+    Py_ssize_t pos;
+    Py_ssize_t string_size;
+    size_t buf_size;
+
+    char ok; /* initialized? */
+    char closed;
+    char readuniversal;
+    char readtranslate;
+    PyObject *decoder;
+    PyObject *readnl;
+    PyObject *writenl;
+    
+    PyObject *dict;
+    PyObject *weakreflist;
+} stringio;
+
+#define CHECK_INITIALIZED(self) \
+    if (self->ok <= 0) { \
+        PyErr_SetString(PyExc_ValueError, \
+            "I/O operation on uninitialized object"); \
+        return NULL; \
+    }
+
+#define CHECK_CLOSED(self) \
+    if (self->closed) { \
+        PyErr_SetString(PyExc_ValueError, \
+            "I/O operation on closed file"); \
+        return NULL; \
+    }
+
+PyDoc_STRVAR(stringio_doc,
+    "Text I/O implementation using an in-memory buffer.\n"
+    "\n"
+    "The initial_value argument sets the value of object.  The newline\n"
+    "argument is like the one of TextIOWrapper's constructor.");
+
+
+/* Internal routine for changing the size, in terms of characters, of the
+   buffer of StringIO objects.  The caller should ensure that the 'size'
+   argument is non-negative.  Returns 0 on success, -1 otherwise. */
+static int
+resize_buffer(stringio *self, size_t size)
+{
+    /* Here, unsigned types are used to avoid dealing with signed integer
+       overflow, which is undefined in C. */
+    size_t alloc = self->buf_size;
+    Py_UNICODE *new_buf = NULL;
+
+    assert(self->buf != NULL);
+
+    /* Reserve one more char for line ending detection. */
+    size = size + 1;
+    /* For simplicity, stay in the range of the signed type. Anyway, Python
+       doesn't allow strings to be longer than this. */
+    if (size > PY_SSIZE_T_MAX)
+        goto overflow;
+
+    if (size < alloc / 2) {
+        /* Major downsize; resize down to exact size. */
+        alloc = size + 1;
+    }
+    else if (size < alloc) {
+        /* Within allocated size; quick exit */
+        return 0;
+    }
+    else if (size <= alloc * 1.125) {
+        /* Moderate upsize; overallocate similar to list_resize() */
+        alloc = size + (size >> 3) + (size < 9 ? 3 : 6);
+    }
+    else {
+        /* Major upsize; resize up to exact size */
+        alloc = size + 1;
+    }
+
+    if (alloc > ((size_t)-1) / sizeof(Py_UNICODE))
+        goto overflow;
+    new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf,
+                                          alloc * sizeof(Py_UNICODE));
+    if (new_buf == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
+    self->buf_size = alloc;
+    self->buf = new_buf;
+
+    return 0;
+
+  overflow:
+    PyErr_SetString(PyExc_OverflowError,
+                    "new buffer size too large");
+    return -1;
+}
+
+/* Internal routine for writing a whole PyUnicode object to the buffer of a
+   StringIO object. Returns 0 on success, or -1 on error. */
+static Py_ssize_t
+write_str(stringio *self, PyObject *obj)
+{
+    Py_UNICODE *str;
+    Py_ssize_t len;
+    PyObject *decoded = NULL;
+    assert(self->buf != NULL);
+    assert(self->pos >= 0);
+
+    if (self->decoder != NULL) {
+        decoded = _PyIncrementalNewlineDecoder_decode(
+            self->decoder, obj, 1 /* always final */);
+    }
+    else {
+        decoded = obj;
+        Py_INCREF(decoded);
+    }
+    if (self->writenl) {
+        PyObject *translated = PyUnicode_Replace(
+            decoded, _PyIO_str_nl, self->writenl, -1);
+        Py_DECREF(decoded);
+        decoded = translated;
+    }
+    if (decoded == NULL)
+        return -1;
+
+    assert(PyUnicode_Check(decoded));
+    str = PyUnicode_AS_UNICODE(decoded);
+    len = PyUnicode_GET_SIZE(decoded);
+
+    assert(len >= 0);
+
+    /* This overflow check is not strictly necessary. However, it avoids us to
+       deal with funky things like comparing an unsigned and a signed
+       integer. */
+    if (self->pos > PY_SSIZE_T_MAX - len) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "new position too large");
+        goto fail;
+    }
+    if (self->pos + len > self->string_size) {
+        if (resize_buffer(self, self->pos + len) < 0)
+            goto fail;
+    }
+
+    if (self->pos > self->string_size) {
+        /* In case of overseek, pad with null bytes the buffer region between
+           the end of stream and the current position.
+
+          0   lo      string_size                           hi
+          |   |<---used--->|<----------available----------->|
+          |   |            <--to pad-->|<---to write--->    |
+          0   buf                   positon
+
+        */
+        memset(self->buf + self->string_size, '\0',
+               (self->pos - self->string_size) * sizeof(Py_UNICODE));
+    }
+
+    /* Copy the data to the internal buffer, overwriting some of the
+       existing data if self->pos < self->string_size. */
+    memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE));
+    self->pos += len;
+
+    /* Set the new length of the internal string if it has changed. */
+    if (self->string_size < self->pos) {
+        self->string_size = self->pos;
+    }
+
+    Py_DECREF(decoded);
+    return 0;
+
+fail:
+    Py_XDECREF(decoded);
+    return -1;
+}
+
+PyDoc_STRVAR(stringio_getvalue_doc,
+    "Retrieve the entire contents of the object.");
+
+static PyObject *
+stringio_getvalue(stringio *self)
+{
+    CHECK_INITIALIZED(self);
+    CHECK_CLOSED(self);
+    return PyUnicode_FromUnicode(self->buf, self->string_size);
+}
+
+PyDoc_STRVAR(stringio_tell_doc,
+    "Tell the current file position.");
+
+static PyObject *
+stringio_tell(stringio *self)
+{
+    CHECK_INITIALIZED(self);
+    CHECK_CLOSED(self);
+    return PyLong_FromSsize_t(self->pos);
+}
+
+PyDoc_STRVAR(stringio_read_doc,
+    "Read at most n characters, returned as a string.\n"
+    "\n"
+    "If the argument is negative or omitted, read until EOF\n"
+    "is reached. Return an empty string at EOF.\n");
+
+static PyObject *
+stringio_read(stringio *self, PyObject *args)
+{
+    Py_ssize_t size, n;
+    Py_UNICODE *output;
+    PyObject *arg = Py_None;
+
+    CHECK_INITIALIZED(self);
+    if (!PyArg_ParseTuple(args, "|O:read", &arg))
+        return NULL;
+    CHECK_CLOSED(self);
+
+    if (PyNumber_Check(arg)) {
+        size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
+        if (size == -1 && PyErr_Occurred())
+            return NULL;
+    }
+    else if (arg == Py_None) {
+        /* Read until EOF is reached, by default. */
+        size = -1;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
+                     Py_TYPE(arg)->tp_name);
+        return NULL;
+    }
+
+    /* adjust invalid sizes */
+    n = self->string_size - self->pos;
+    if (size < 0 || size > n) {
+        size = n;
+        if (size < 0)
+            size = 0;
+    }
+
+    output = self->buf + self->pos;
+    self->pos += size;
+    return PyUnicode_FromUnicode(output, size);
+}
+
+/* Internal helper, used by stringio_readline and stringio_iternext */
+static PyObject *
+_stringio_readline(stringio *self, Py_ssize_t limit)
+{
+    Py_UNICODE *start, *end, old_char;
+    Py_ssize_t len, consumed;
+
+    /* In case of overseek, return the empty string */
+    if (self->pos >= self->string_size)
+        return PyUnicode_FromString("");
+
+    start = self->buf + self->pos;
+    if (limit < 0 || limit > self->string_size - self->pos)
+        limit = self->string_size - self->pos;
+
+    end = start + limit;
+    old_char = *end;
+    *end = '\0';
+    len = _PyIO_find_line_ending(
+        self->readtranslate, self->readuniversal, self->readnl,
+        start, end, &consumed);
+    *end = old_char;
+    /* If we haven't found any line ending, we just return everything
+       (`consumed` is ignored). */
+    if (len < 0)
+        len = limit;
+    self->pos += len;
+    return PyUnicode_FromUnicode(start, len);
+}
+
+PyDoc_STRVAR(stringio_readline_doc,
+    "Read until newline or EOF.\n"
+    "\n"
+    "Returns an empty string if EOF is hit immediately.\n");
+
+static PyObject *
+stringio_readline(stringio *self, PyObject *args)
+{
+    PyObject *arg = Py_None;
+    Py_ssize_t limit = -1;
+
+    CHECK_INITIALIZED(self);
+    if (!PyArg_ParseTuple(args, "|O:readline", &arg))
+        return NULL;
+    CHECK_CLOSED(self);
+
+    if (PyNumber_Check(arg)) {
+        limit = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
+        if (limit == -1 && PyErr_Occurred())
+            return NULL;
+    }
+    else if (arg != Py_None) {
+        PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
+                     Py_TYPE(arg)->tp_name);
+        return NULL;
+    }
+    return _stringio_readline(self, limit);
+}
+
+static PyObject *
+stringio_iternext(stringio *self)
+{
+    PyObject *line;
+
+    CHECK_INITIALIZED(self);
+    CHECK_CLOSED(self);
+
+    if (Py_TYPE(self) == &PyStringIO_Type) {
+        /* Skip method call overhead for speed */
+        line = _stringio_readline(self, -1);
+    }
+    else {
+        /* XXX is subclassing StringIO really supported? */
+        line = PyObject_CallMethodObjArgs((PyObject *)self,
+                                           _PyIO_str_readline, NULL);
+        if (line && !PyUnicode_Check(line)) {
+            PyErr_Format(PyExc_IOError,
+                         "readline() should have returned an str object, "
+                         "not '%.200s'", Py_TYPE(line)->tp_name);
+            Py_DECREF(line);
+            return NULL;
+        }
+    }
+
+    if (line == NULL)
+        return NULL;
+
+    if (PyUnicode_GET_SIZE(line) == 0) {
+        /* Reached EOF */
+        Py_DECREF(line);
+        return NULL;
+    }
+
+    return line;
+}
+
+PyDoc_STRVAR(stringio_truncate_doc,
+    "Truncate size to pos.\n"
+    "\n"
+    "The pos argument defaults to the current file position, as\n"
+    "returned by tell().  Imply an absolute seek to pos.\n"
+    "Returns the new absolute position.\n");
+
+static PyObject *
+stringio_truncate(stringio *self, PyObject *args)
+{
+    Py_ssize_t size;
+    PyObject *arg = Py_None;
+
+    CHECK_INITIALIZED(self);
+    if (!PyArg_ParseTuple(args, "|O:truncate", &arg))
+        return NULL;
+    CHECK_CLOSED(self);
+
+    if (PyNumber_Check(arg)) {
+        size = PyNumber_AsSsize_t(arg, PyExc_OverflowError);
+        if (size == -1 && PyErr_Occurred())
+            return NULL;
+    }
+    else if (arg == Py_None) {
+        /* Truncate to current position if no argument is passed. */
+        size = self->pos;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'",
+                     Py_TYPE(arg)->tp_name);
+        return NULL;
+    }
+
+    if (size < 0) {
+        PyErr_Format(PyExc_ValueError,
+                     "Negative size value %zd", size);
+        return NULL;
+    }
+
+    if (size < self->string_size) {
+        if (resize_buffer(self, size) < 0)
+            return NULL;
+        self->string_size = size;
+    }
+    self->pos = size;
+
+    return PyLong_FromSsize_t(size);
+}
+
+PyDoc_STRVAR(stringio_seek_doc,
+    "Change stream position.\n"
+    "\n"
+    "Seek to character offset pos relative to position indicated by whence:\n"
+    "    0  Start of stream (the default).  pos should be >= 0;\n"
+    "    1  Current position - pos must be 0;\n"
+    "    2  End of stream - pos must be 0.\n"
+    "Returns the new absolute position.\n");
+
+static PyObject *
+stringio_seek(stringio *self, PyObject *args)
+{
+    PyObject *posobj;
+    Py_ssize_t pos;
+    int mode = 0;
+
+    CHECK_INITIALIZED(self);
+    if (!PyArg_ParseTuple(args, "O|i:seek", &posobj, &mode))
+        return NULL;
+
+    pos = PyNumber_AsSsize_t(posobj, PyExc_OverflowError);
+    if (pos == -1 && PyErr_Occurred())
+        return NULL;
+    
+    CHECK_CLOSED(self);
+
+    if (mode != 0 && mode != 1 && mode != 2) {
+        PyErr_Format(PyExc_ValueError,
+                     "Invalid whence (%i, should be 0, 1 or 2)", mode);
+        return NULL;
+    }
+    else if (pos < 0 && mode == 0) {
+        PyErr_Format(PyExc_ValueError,
+                     "Negative seek position %zd", pos);
+        return NULL;
+    }
+    else if (mode != 0 && pos != 0) {
+        PyErr_SetString(PyExc_IOError,
+                        "Can't do nonzero cur-relative seeks");
+        return NULL;
+    }
+
+    /* mode 0: offset relative to beginning of the string.
+       mode 1: no change to current position.
+       mode 2: change position to end of file. */
+    if (mode == 1) {
+        pos = self->pos;
+    }
+    else if (mode == 2) {
+        pos = self->string_size;
+    }
+
+    self->pos = pos;
+
+    return PyLong_FromSsize_t(self->pos);
+}
+
+PyDoc_STRVAR(stringio_write_doc,
+    "Write string to file.\n"
+    "\n"
+    "Returns the number of characters written, which is always equal to\n"
+    "the length of the string.\n");
+
+static PyObject *
+stringio_write(stringio *self, PyObject *obj)
+{
+    Py_ssize_t size;
+
+    CHECK_INITIALIZED(self);
+    if (!PyUnicode_Check(obj)) {
+        PyErr_Format(PyExc_TypeError, "string argument expected, got '%s'",
+                     Py_TYPE(obj)->tp_name);
+        return NULL;
+    }
+    CHECK_CLOSED(self);
+    size = PyUnicode_GET_SIZE(obj);
+
+    if (size > 0 && write_str(self, obj) < 0)
+        return NULL;
+
+    return PyLong_FromSsize_t(size);
+}
+
+PyDoc_STRVAR(stringio_close_doc,
+    "Close the IO object. Attempting any further operation after the\n"
+    "object is closed will raise a ValueError.\n"
+    "\n"
+    "This method has no effect if the file is already closed.\n");
+
+static PyObject *
+stringio_close(stringio *self)
+{
+    self->closed = 1;
+    /* Free up some memory */
+    if (resize_buffer(self, 0) < 0)
+        return NULL;
+    Py_CLEAR(self->readnl);
+    Py_CLEAR(self->writenl);
+    Py_CLEAR(self->decoder);
+    Py_RETURN_NONE;
+}
+
+static int
+stringio_traverse(stringio *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->dict);
+    return 0;
+}
+
+static int
+stringio_clear(stringio *self)
+{
+    Py_CLEAR(self->dict);
+    return 0;
+}
+
+static void
+stringio_dealloc(stringio *self)
+{
+    _PyObject_GC_UNTRACK(self);
+    Py_CLEAR(self->readnl);
+    Py_CLEAR(self->writenl);
+    Py_CLEAR(self->decoder);
+    if (self->buf)
+        PyMem_Free(self->buf);
+    if (self->weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) self);
+    Py_TYPE(self)->tp_free(self);
+}
+
+static PyObject *
+stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    stringio *self;
+
+    assert(type != NULL && type->tp_alloc != NULL);
+    self = (stringio *)type->tp_alloc(type, 0);
+    if (self == NULL)
+        return NULL;
+
+    self->string_size = 0;
+    self->pos = 0;
+    self->buf_size = 0;
+    self->buf = (Py_UNICODE *)PyMem_Malloc(0);
+    if (self->buf == NULL) {
+        Py_DECREF(self);
+        return PyErr_NoMemory();
+    }
+
+    return (PyObject *)self;
+}
+
+static int
+stringio_init(stringio *self, PyObject *args, PyObject *kwds)
+{
+    char *kwlist[] = {"initial_value", "newline", NULL};
+    PyObject *value = NULL;
+    char *newline = "\n";
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oz:__init__", kwlist,
+                                     &value, &newline))
+        return -1;
+
+    if (newline && newline[0] != '\0'
+        && !(newline[0] == '\n' && newline[1] == '\0')
+        && !(newline[0] == '\r' && newline[1] == '\0')
+        && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) {
+        PyErr_Format(PyExc_ValueError,
+                     "illegal newline value: %s", newline);
+        return -1;
+    }
+    if (value && value != Py_None && !PyUnicode_Check(value)) {
+        PyErr_Format(PyExc_ValueError,
+                     "initial_value must be str or None, not %.200s",
+                     Py_TYPE(value)->tp_name);
+        return -1;
+    }
+
+    self->ok = 0;
+
+    Py_CLEAR(self->readnl);
+    Py_CLEAR(self->writenl);
+    Py_CLEAR(self->decoder);
+
+    if (newline) {
+        self->readnl = PyString_FromString(newline);
+        if (self->readnl == NULL)
+            return -1;
+    }
+    self->readuniversal = (newline == NULL || newline[0] == '\0');
+    self->readtranslate = (newline == NULL);
+    /* If newline == "", we don't translate anything.
+       If newline == "\n" or newline == None, we translate to "\n", which is
+       a no-op.
+       (for newline == None, TextIOWrapper translates to os.sepline, but it
+       is pointless for StringIO)
+    */
+    if (newline != NULL && newline[0] == '\r') {
+        self->writenl = PyUnicode_FromString(newline);
+    }
+
+    if (self->readuniversal) {
+        self->decoder = PyObject_CallFunction(
+            (PyObject *)&PyIncrementalNewlineDecoder_Type,
+            "Oi", Py_None, (int) self->readtranslate);
+        if (self->decoder == NULL)
+            return -1;
+    }
+
+    /* Now everything is set up, resize buffer to size of initial value,
+       and copy it */
+    self->string_size = 0;
+    if (value && value != Py_None) {
+        Py_ssize_t len = PyUnicode_GetSize(value);
+        /* This is a heuristic, for newline translation might change
+           the string length. */
+        if (resize_buffer(self, len) < 0)
+            return -1;
+        self->pos = 0;
+        if (write_str(self, value) < 0)
+            return -1;
+    }
+    else {
+        if (resize_buffer(self, 0) < 0)
+            return -1;
+    }
+    self->pos = 0;
+
+    self->closed = 0;
+    self->ok = 1;
+    return 0;
+}
+
+/* Properties and pseudo-properties */
+static PyObject *
+stringio_seekable(stringio *self, PyObject *args)
+{
+    CHECK_INITIALIZED(self);
+    Py_RETURN_TRUE;
+}
+
+static PyObject *
+stringio_readable(stringio *self, PyObject *args)
+{
+    CHECK_INITIALIZED(self);
+    Py_RETURN_TRUE;
+}
+
+static PyObject *
+stringio_writable(stringio *self, PyObject *args)
+{
+    CHECK_INITIALIZED(self);
+    Py_RETURN_TRUE;
+}
+
+static PyObject *
+stringio_buffer(stringio *self, void *context)
+{
+    PyErr_SetString(_PyIO_unsupported_operation,
+                    "buffer attribute is unsupported on type StringIO");
+    return NULL;
+}
+
+static PyObject *
+stringio_closed(stringio *self, void *context)
+{
+    CHECK_INITIALIZED(self);
+    return PyBool_FromLong(self->closed);
+}
+
+static PyObject *
+stringio_line_buffering(stringio *self, void *context)
+{
+    CHECK_INITIALIZED(self);
+    CHECK_CLOSED(self);
+    Py_RETURN_FALSE;
+}
+
+static PyObject *
+stringio_newlines(stringio *self, void *context)
+{
+    CHECK_INITIALIZED(self);
+    CHECK_CLOSED(self);
+    if (self->decoder == NULL)
+        Py_RETURN_NONE;
+    return PyObject_GetAttr(self->decoder, _PyIO_str_newlines);
+}
+
+static struct PyMethodDef stringio_methods[] = {
+    {"close",    (PyCFunction)stringio_close,    METH_NOARGS,  stringio_close_doc},
+    {"getvalue", (PyCFunction)stringio_getvalue, METH_VARARGS, stringio_getvalue_doc},
+    {"read",     (PyCFunction)stringio_read,     METH_VARARGS, stringio_read_doc},
+    {"readline", (PyCFunction)stringio_readline, METH_VARARGS, stringio_readline_doc},
+    {"tell",     (PyCFunction)stringio_tell,     METH_NOARGS,  stringio_tell_doc},
+    {"truncate", (PyCFunction)stringio_truncate, METH_VARARGS, stringio_truncate_doc},
+    {"seek",     (PyCFunction)stringio_seek,     METH_VARARGS, stringio_seek_doc},
+    {"write",    (PyCFunction)stringio_write,    METH_O,       stringio_write_doc},
+    
+    {"seekable", (PyCFunction)stringio_seekable, METH_NOARGS},
+    {"readable", (PyCFunction)stringio_readable, METH_NOARGS},
+    {"writable", (PyCFunction)stringio_writable, METH_NOARGS},
+    {NULL, NULL}        /* sentinel */
+};
+
+static PyGetSetDef stringio_getset[] = {
+    {"closed",         (getter)stringio_closed,         NULL, NULL},
+    {"newlines",       (getter)stringio_newlines,       NULL, NULL},
+    /*  (following comments straight off of the original Python wrapper:)
+        XXX Cruft to support the TextIOWrapper API. This would only
+        be meaningful if StringIO supported the buffer attribute.
+        Hopefully, a better solution, than adding these pseudo-attributes,
+        will be found.
+    */
+    {"buffer",         (getter)stringio_buffer,         NULL, NULL},
+    {"line_buffering", (getter)stringio_line_buffering, NULL, NULL},
+    {NULL}
+};
+
+PyTypeObject PyStringIO_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_io.StringIO",                            /*tp_name*/
+    sizeof(stringio),                    /*tp_basicsize*/
+    0,                                         /*tp_itemsize*/
+    (destructor)stringio_dealloc,              /*tp_dealloc*/
+    0,                                         /*tp_print*/
+    0,                                         /*tp_getattr*/
+    0,                                         /*tp_setattr*/
+    0,                                         /*tp_reserved*/
+    0,                                         /*tp_repr*/
+    0,                                         /*tp_as_number*/
+    0,                                         /*tp_as_sequence*/
+    0,                                         /*tp_as_mapping*/
+    0,                                         /*tp_hash*/
+    0,                                         /*tp_call*/
+    0,                                         /*tp_str*/
+    0,                                         /*tp_getattro*/
+    0,                                         /*tp_setattro*/
+    0,                                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
+                       | Py_TPFLAGS_HAVE_GC,   /*tp_flags*/
+    stringio_doc,                              /*tp_doc*/
+    (traverseproc)stringio_traverse,           /*tp_traverse*/
+    (inquiry)stringio_clear,                   /*tp_clear*/
+    0,                                         /*tp_richcompare*/
+    offsetof(stringio, weakreflist),            /*tp_weaklistoffset*/
+    0,                                         /*tp_iter*/
+    (iternextfunc)stringio_iternext,           /*tp_iternext*/
+    stringio_methods,                          /*tp_methods*/
+    0,                                         /*tp_members*/
+    stringio_getset,                           /*tp_getset*/
+    0,                                         /*tp_base*/
+    0,                                         /*tp_dict*/
+    0,                                         /*tp_descr_get*/
+    0,                                         /*tp_descr_set*/
+    offsetof(stringio, dict),                  /*tp_dictoffset*/
+    (initproc)stringio_init,                   /*tp_init*/
+    0,                                         /*tp_alloc*/
+    stringio_new,                              /*tp_new*/
+};

setup.py

@@ -438,10 +438,11 @@ class PyBuildExt(build_ext):
         exts.append( Extension("_heapq", ["_heapqmodule.c"]) )
         # operator.add() and similar goodies
         exts.append( Extension('operator', ['operator.c']) )
-        # Python 3.0 _fileio module
+        # Python 3.1 _io library
-        exts.append( Extension("_fileio", ["_fileio.c"]) )
+        exts.append( Extension("_io",
-        # Python 3.0 _bytesio module
+            ["_io/bufferedio.c", "_io/bytesio.c", "_io/fileio.c",
-        exts.append( Extension("_bytesio", ["_bytesio.c"]) )
+             "_io/iobase.c", "_io/_iomodule.c", "_io/stringio.c", "_io/textio.c"],
+             depends=["_io/_iomodule.h"], include_dirs=["Modules/_io"]))
         # _functools
         exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
         # _json speedups