Revert r63934 -- it was mixing two patches.

Doc/c-api/unicode.rst

@@ -144,13 +144,6 @@ the Python configuration.
 
    Return 1 or 0 depending on whether *ch* is an alphanumeric character.
 
-.. cfunction:: int Py_UNICODE_ISPRINTABLE(Py_UNICODE ch)
-
-   Return 1 or 0 depending on whether *ch* is a printable character.
-   Characters defined in the Unicode character database as "Other"
-   or "Separator" other than ASCII space(0x20) are not considered
-   printable.
-
 These APIs can be used for fast direct character conversions:
 
 
@@ -235,9 +228,6 @@ APIs:
    +===================+=====================+================================+
    | :attr:`%%`        | *n/a*               | The literal % character.       |
    +-------------------+---------------------+--------------------------------+
-   | :attr:`%a`        | PyObject\*          | The result of calling          |
-   |                   |                     | :func:`ascii`.                 |
-   +-------------------+---------------------+--------------------------------+
    | :attr:`%c`        | int                 | A single character,            |
    |                   |                     | represented as an C int.       |
    +-------------------+---------------------+--------------------------------+

Doc/library/functions.rst

@@ -91,14 +91,6 @@ are always available.  They are listed here in alphabetical order.
           return False
 
 
-.. function:: ascii(object)
-
-   As :func:`repr`, return a string containing a printable
-   representation of an object. But unlike :func:`repr`, the non-ASCII
-   characters in the string returned by :func:`ascii`() are hex-escaped
-   to generate a same string as :func:`repr` in Python 2.
-
-
 .. function:: bin(x)
 
    Convert an integer number to a binary string. The result is a valid Python

Doc/library/stdtypes.rst

@@ -774,14 +774,6 @@ functions based on regular expressions.
    least one cased character, false otherwise.
 
 
-.. method:: str.isprintable()
-
-   Return true if all characters in the string are printable and there is at
-   least one character, false otherwise. Characters defined in the Unicode 
-   character database as "Other" or "Separator" other than ASCII space(0x20) are 
-   not considered printable.
-
-
 .. method:: str.isspace()
 
    Return true if there are only whitespace characters in the string and there is

Doc/library/sys.rst

@@ -231,6 +231,8 @@ always available.
    +------------------------------+------------------------------------------+
    | :const:`ignore_environment`  | -E                                       |
    +------------------------------+------------------------------------------+
+   | :const:`tabcheck`            | -t or -tt                                |
+   +------------------------------+------------------------------------------+
    | :const:`verbose`             | -v                                       |
    +------------------------------+------------------------------------------+
    | :const:`unicode`             | -U                                       |

Doc/using/cmdline.rst

@@ -222,6 +222,13 @@ Miscellaneous options
    manipulations of :data:`sys.path` that it entails.
 
 
+.. cmdoption:: -t
+
+   Issue a warning when a source file mixes tabs and spaces for indentation in a
+   way that makes it depend on the worth of a tab expressed in spaces.  Issue an
+   error when the option is given twice (:option:`-tt`).
+
+
 .. cmdoption:: -u
    
    Force stdin, stdout and stderr to be totally unbuffered.  On systems where it

Include/pydebug.h

@@ -14,6 +14,7 @@ PyAPI_DATA(int) Py_NoSiteFlag;
 PyAPI_DATA(int) Py_BytesWarningFlag;
 PyAPI_DATA(int) Py_UseClassExceptionsFlag;
 PyAPI_DATA(int) Py_FrozenFlag;
+PyAPI_DATA(int) Py_TabcheckFlag;
 PyAPI_DATA(int) Py_IgnoreEnvironmentFlag;
 PyAPI_DATA(int) Py_DivisionWarningFlag;
 PyAPI_DATA(int) Py_DontWriteBytecodeFlag;

Include/unicodeobject.h

@@ -217,7 +217,6 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
 # define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
 # define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
-# define _PyUnicode_IsPrintable _PyUnicodeUCS2_IsPrintable
 # define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
 # define _PyUnicode_IsXidStart _PyUnicodeUCS2_IsXidStart
 # define _PyUnicode_IsXidContinue _PyUnicodeUCS2_IsXidContinue
@@ -312,7 +311,6 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
 # define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
 # define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
-# define _PyUnicode_IsPrintable _PyUnicodeUCS4_IsPrintable
 # define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
 # define _PyUnicode_IsXidStart _PyUnicodeUCS4_IsXidStart
 # define _PyUnicode_IsXidContinue _PyUnicodeUCS4_IsXidContinue
@@ -353,7 +351,6 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
-#define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch)
 
 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
@@ -384,7 +381,6 @@ extern const unsigned char _Py_ascii_whitespace[];
 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
-#define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch)
 
 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
@@ -1503,10 +1499,6 @@ PyAPI_FUNC(int) _PyUnicode_IsNumeric(
     Py_UNICODE ch 	/* Unicode character */
     );
 
-PyAPI_FUNC(int) _PyUnicode_IsPrintable(
-    Py_UNICODE ch 	/* Unicode character */
-    );
-
 PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     Py_UNICODE ch 	/* Unicode character */
     );

Lib/doctest.py

@@ -1441,13 +1441,6 @@ class OutputChecker:
     and returns true if they match; and `output_difference`, which
     returns a string describing the differences between two outputs.
     """
-
-    def _toAscii(self, s):
-        """
-        Convert string to hex-escaped ASCII string.
-        """
-        return str(s.encode('ASCII', 'backslashreplace'), "ASCII")
-
     def check_output(self, want, got, optionflags):
         """
         Return True iff the actual output from an example (`got`)
@@ -1458,15 +1451,6 @@ class OutputChecker:
         documentation for `TestRunner` for more information about
         option flags.
         """
-
-        # If `want` contains hex-escaped character such as "\u1234",
-        # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]).
-        # On the other hand, `got` could be an another sequence of
-        # characters such as [\u1234], so `want` and `got` should
-        # be folded to hex-escaped ASCII string to compare.
-        got = self._toAscii(got)
-        want = self._toAscii(want)
-
         # Handle the common case first, for efficiency:
         # if they're string-identical, always return true.
         if got == want:

Lib/test/test_array.py

@@ -768,7 +768,7 @@ class UnicodeTest(StringTest):
         a = array.array('u', s)
         self.assertEqual(
             repr(a),
-            "array('u', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')")
+            "array('u', '\\x00=\"\\'a\\\\b\\x80\\xff\\x00\\x01\\u1234')")
 
         self.assertRaises(TypeError, a.fromunicode)
 

Lib/test/test_builtin.py

@@ -159,9 +159,6 @@ class BuiltinTest(unittest.TestCase):
         S = [10, 20, 30]
         self.assertEqual(any(x > 42 for x in S), False)
 
-    def test_ascii(self):
-        self.assertEqual(ascii("\u0370"), "'\\u0370'")
-
     def test_neg(self):
         x = -sys.maxsize-1
         self.assert_(isinstance(x, int))

Lib/test/test_exceptions.py

@@ -80,10 +80,10 @@ class ExceptionTests(unittest.TestCase):
         self.raise_catch(IndentationError, "IndentationError")
 
         self.raise_catch(TabError, "TabError")
-        try: compile("try:\n\t1/0\n    \t1/0\nfinally:\n pass\n",
+        # can only be tested under -tt, and is the only test for -tt
-                     '<string>', 'exec')
+        #try: compile("try:\n\t1/0\n    \t1/0\nfinally:\n pass\n", '<string>', 'exec')
-        except TabError: pass
+        #except TabError: pass
-        else: self.fail("TabError not raised")
+        #else: self.fail("TabError not raised")
 
         self.raise_catch(SystemError, "SystemError")
 

Lib/test/test_format.py

@@ -216,8 +216,6 @@ class FormatTest(unittest.TestCase):
         testformat("%o", 0o42, "42")
         testformat("%o", -0o42, "-42")
         testformat("%o", float(0o42), "42")
-        testformat("%r", "\u0370", "'\u0370'")
-        testformat("%a", "\u0370", "'\\u0370'")
         # Test exception for unknown format characters
         if verbose:
             print('Testing exceptions')
@@ -237,8 +235,8 @@ class FormatTest(unittest.TestCase):
                 raise
             else:
                 raise TestFailed('did not get expected exception: %s' % excmsg)
-        test_exc('abc %b', 1, ValueError,
+        test_exc('abc %a', 1, ValueError,
-                 "unsupported format character 'b' (0x62) at index 5")
+                 "unsupported format character 'a' (0x61) at index 5")
         #test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError,
         #         "unsupported format character '?' (0x3000) at index 5")
         test_exc('%d', '1', TypeError, "%d format: a number is required, not str")

Lib/test/test_pyexpat.py

@@ -131,7 +131,7 @@ class ParseTest(unittest.TestCase):
         self.assertEquals(op[1], "Comment: ' comment data '")
         self.assertEquals(op[2], "Notation declared: ('notation', None, 'notation.jpeg', None)")
         self.assertEquals(op[3], "Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')")
-        self.assertEquals(op[4], "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\u1f40'}")
+        self.assertEquals(op[4], "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\\u1f40'}")
         self.assertEquals(op[5], "NS decl: 'myns' 'http://www.python.org/namespace'")
         self.assertEquals(op[6], "Start element: 'http://www.python.org/namespace!subelement' {}")
         self.assertEquals(op[7], "Character data: 'Contents of subelements'")

Lib/test/test_unicode.py

@@ -94,14 +94,14 @@ class UnicodeTest(
                 "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f"
                 "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d"
                 "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b"
-                "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9"
+                "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9"
-                "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+                "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7"
-                "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5"
+                "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5"
-                "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3"
+                "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3"
-                "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1"
+                "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1"
-                "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+                "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef"
-                "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd"
+                "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd"
-                "\xfe\xff'")
+                "\\xfe\\xff'")
             testrepr = repr(''.join(map(chr, range(256))))
             self.assertEqual(testrepr, latin1repr)
             # Test repr works on wide unicode escapes without overflow.
@@ -374,12 +374,6 @@ class UnicodeTest(
         self.assertFalse("[".isidentifier())
         self.assertFalse("©".isidentifier())
 
-    def test_isprintable(self):
-        self.assertTrue("abcdefg".isprintable())
-        self.assertFalse("abcdefg\n".isprintable())
-        self.assertTrue("\u0370".isprintable())
-        self.assertFalse("\ud800".isprintable())
-
     def test_contains(self):
         # Testing Unicode contains method
         self.assert_('a' in 'abdb')

Mac/BuildScript/scripts/postflight.framework

@@ -6,22 +6,22 @@
 PYVER="@PYVER@"
 FWK="/Library/Frameworks/Python.framework/Versions/@PYVER@"
 
-"${FWK}/bin/python" -Wi \
+"${FWK}/bin/python" -Wi -tt \
     "${FWK}/lib/python${PYVER}/compileall.py" \
     -x badsyntax -x site-packages \
     "${FWK}/lib/python${PYVER}"
 
-"${FWK}/bin/python" -Wi -O \
+"${FWK}/bin/python" -Wi -tt -O \
     "${FWK}/lib/python${PYVER}/compileall.py" \
     -x badsyntax -x site-packages \
     "${FWK}/lib/python${PYVER}"
 
-"${FWK}/bin/python" -Wi \
+"${FWK}/bin/python" -Wi -tt \
     "${FWK}/lib/python${PYVER}/compileall.py" \
     -x badsyntax -x site-packages \
     "${FWK}/Mac/Tools"
 
-"${FWK}/bin/python" -Wi -O \
+"${FWK}/bin/python" -Wi -tt -O \
     "${FWK}/lib/python${PYVER}/compileall.py" \
     -x badsyntax -x site-packages \
     "${FWK}/Mac/Tools"

Mac/Makefile.in

@@ -226,8 +226,8 @@ installmacsubtree:
 
 
 	$(RUNSHARED) $(BUILDPYTHON) $(CACHERSRC) -v $(DESTDIR)$(MACLIBDEST) $(DESTDIR)$(MACTOOLSDEST)
-	$(RUNSHARED) $(BUILDPYTHON) -Wi $(compileall) -d $(MACTOOLSDEST) -x badsyntax $(DESTDIR)$(MACTOOLSDEST)
+	$(RUNSHARED) $(BUILDPYTHON) -Wi -tt $(compileall) -d $(MACTOOLSDEST) -x badsyntax $(DESTDIR)$(MACTOOLSDEST)
-	$(RUNSHARED) $(BUILDPYTHON) -O -Wi $(compileall) -d $(MACTOOLSDEST) -x badsyntax $(DESTDIR)$(MACTOOLSDEST)
+	$(RUNSHARED) $(BUILDPYTHON) -O -Wi -tt $(compileall) -d $(MACTOOLSDEST) -x badsyntax $(DESTDIR)$(MACTOOLSDEST)
 
 $(INSTALLED_PYTHONAPP): install_Python
 

Makefile.pre.in

@@ -664,7 +664,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Modules/python.o: $(PYTHON_HEADERS)
 
 TESTOPTS=	-l $(EXTRATESTOPTS)
 TESTPROG=	$(srcdir)/Lib/test/regrtest.py
-TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) -E -bb
+TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) -E -tt -bb
 test:		all platform
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
 		-$(TESTPYTHON) $(TESTPROG) $(TESTOPTS)
@@ -687,7 +687,7 @@ testuniversal:	all platform
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
 		-$(TESTPYTHON) $(TESTPROG) $(TESTOPTS) -uall
 		$(TESTPYTHON) $(TESTPROG) $(TESTOPTS) -uall
-		$(RUNSHARED) /usr/libexec/oah/translate ./$(BUILDPYTHON) -E $(TESTPROG) $(TESTOPTS) -uall
+		$(RUNSHARED) /usr/libexec/oah/translate ./$(BUILDPYTHON) -E -tt $(TESTPROG) $(TESTOPTS) -uall
 
 
 # Like testall, but with a single pass only
@@ -872,23 +872,23 @@ libinstall:	build_all $(srcdir)/Lib/$(PLATDIR)
 	done
 	$(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
+		./$(BUILDPYTHON) -Wi -tt $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages' $(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
+		./$(BUILDPYTHON) -Wi -tt -O $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages' $(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
+		./$(BUILDPYTHON) -Wi -t $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
+		./$(BUILDPYTHON) -Wi -t -O $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -c "import lib2to3.pygram"
+		./$(BUILDPYTHON) -Wi -t -c "import lib2to3.pygram"
 
 # Create the PLATDIR source directory, if one wasn't distributed..
 $(srcdir)/Lib/$(PLATDIR):

Misc/NEWS

@@ -12,8 +12,6 @@ What's new in Python 3.0b1?
 Core and Builtins
 -----------------
 
-- Removed the already-defunct ``-t`` option.
-
 - Issue #2957: Corrected a ValueError "recursion limit exceeded", when
   unmarshalling many code objects, which happens when importing a
   large .pyc file (~1000 functions).

Misc/build.sh

@@ -59,7 +59,7 @@ RSYNC_OPTS="-aC -e ssh"
 PYTHON=$INSTALL_DIR/bin/python
 
 # Python options and regression test program that should always be run.
-REGRTEST_ARGS="-E $INSTALL_DIR/lib/python3.0/test/regrtest.py"
+REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python3.0/test/regrtest.py"
 
 REFLOG="build/reflog.txt.out"
 # These tests are not stable and falsely report leaks sometimes.

Misc/cheatsheet

@@ -46,6 +46,7 @@ Option                                  Effect
 -OO     remove doc-strings in addition to the -O optimizations
 -Q arg  division options: -Qold (default), -Qwarn, -Qwarnall, -Qnew
 -S      Don't perform 'import site' on initialization
+-t      Issue warnings about inconsistent tab usage (-tt: issue errors)
 -u      Unbuffered binary stdout and stderr (also PYTHONUNBUFFERED=x).
 -v      Verbose (trace import statements) (also PYTHONVERBOSE=x)
 -W arg : warning control (arg is action:message:category:module:lineno)

Misc/python.man

@@ -35,6 +35,9 @@ python \- an interpreted, interactive, object-oriented programming language
 .B \-S
 ]
 [
+.B \-t
+]
+[
 .B \-u
 ]
 .br
@@ -141,6 +144,11 @@ and the site-dependent manipulations of
 .I sys.path
 that it entails.
 .TP
+.B \-t
+Issue a warning when a source file mixes tabs and spaces for
+indentation in a way that makes it depend on the worth of a tab
+expressed in spaces.  Issue an error when the option is given twice.
+.TP
 .B \-u
 Force stdin, stdout and stderr to be totally unbuffered.  On systems
 where it matters, also put stdin, stdout and stderr in binary mode.

Misc/valgrind-python.supp

@@ -5,7 +5,7 @@
 #
 #	cd python/dist/src
 #	valgrind --tool=memcheck --suppressions=Misc/valgrind-python.supp \
-#		./python -E ./Lib/test/regrtest.py -u bsddb,network
+#		./python -E -tt ./Lib/test/regrtest.py -u bsddb,network
 #
 # You must edit Objects/obmalloc.c and uncomment Py_USING_MEMORY_DEBUGGER
 # to use the preferred suppressions with Py_ADDRESS_IN_RANGE.

Modules/main.c

@@ -72,6 +72,7 @@ static char *usage_2 = "\
 -OO    : remove doc-strings in addition to the -O optimizations\n\
 -s     : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\
 -S     : don't imply 'import site' on initialization\n\
+-t     : issue warnings about inconsistent tab usage (-tt: issue errors)\n\
 ";
 static char *usage_3 = "\
 -u     : unbuffered binary stdout and stderr; also PYTHONUNBUFFERED=x\n\
@@ -369,7 +370,7 @@ Py_Main(int argc, wchar_t **argv)
 			break;
 
 		case 't':
-			/* ignored for backwards compatibility */
+			Py_TabcheckFlag++;
 			break;
 
 		case 'u':

Objects/unicodectype.c

@@ -21,7 +21,6 @@
 #define UPPER_MASK 0x80
 #define XID_START_MASK 0x100
 #define XID_CONTINUE_MASK 0x200
-#define NONPRINTABLE_MASK 0x400
 
 typedef struct {
     const Py_UNICODE upper;
@@ -676,26 +675,6 @@ int _PyUnicode_IsNumeric(Py_UNICODE ch)
     return _PyUnicode_ToNumeric(ch) != -1.0;
 }
 
-/* Returns 1 for Unicode characters to be hex-escaped when repr()ed,
-   0 otherwise.
-   Characters defined in the Unicode character database as following
-   categories are not considered printable.
-      * Cc (Other, Control)
-      * Cf (Other, Format)
-      * Cs (Other, Surrogate)
-      * Co (Other, Private Use)
-      * Cn (Other, Not Assigned)
-      * Zl Separator, Line ('\u2028', LINE SEPARATOR)
-      * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
-      * Zs (Separator, Space) other than ASCII space('\x20').
-*/
-int _PyUnicode_IsPrintable(Py_UNICODE ch)
-{
-    const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
-
-    return (ctype->flags & NONPRINTABLE_MASK) == 0;
-}
-
 #ifndef WANT_WCTYPE_FUNCTIONS
 
 /* Returns 1 for Unicode characters having the bidirectional type

Objects/unicodeobject.c

@@ -7149,36 +7149,6 @@ unicode_isidentifier(PyObject *self)
     return PyBool_FromLong(PyUnicode_IsIdentifier(self));
 }
 
-PyDoc_STRVAR(isprintable__doc__,
-"S.isprintable() -> bool\n\
-\n\
-Return True if all characters in S are considered\n\
-printable in repr() and there is at least one character\n\
-in S, False otherwise.");
-
-static PyObject*
-unicode_isprintable(PyObject *self)
-{
-    register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
-    register const Py_UNICODE *e;
-
-    /* Shortcut for single character strings */
-    if (PyUnicode_GET_SIZE(self) == 1 &&
-	Py_UNICODE_ISPRINTABLE(*p))
-	return PyBool_FromLong(1);
-
-    /* Special case for empty strings */
-    if (PyUnicode_GET_SIZE(self) == 0)
-	return PyBool_FromLong(0);
-
-    e = p + PyUnicode_GET_SIZE(self);
-    for (; p < e; p++) {
-	if (!Py_UNICODE_ISPRINTABLE(*p))
-	    return PyBool_FromLong(0);
-    }
-    return PyBool_FromLong(1);
-}
-
 PyDoc_STRVAR(join__doc__,
 "S.join(sequence) -> str\n\
 \n\
@@ -7556,8 +7526,61 @@ PyObject *unicode_repr(PyObject *unicode)
             continue;
         }
 
-	/* Map special whitespace to '\t', \n', '\r' */
+#ifdef Py_UNICODE_WIDE
-        if (ch == '\t') {
+        /* Map 21-bit characters to '\U00xxxxxx' */
+        else if (ch >= 0x10000) {
+            *p++ = '\\';
+            *p++ = 'U';
+            *p++ = hexdigits[(ch >> 28) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 24) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 20) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 16) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 12) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 8) & 0x0000000F];
+            *p++ = hexdigits[(ch >> 4) & 0x0000000F];
+            *p++ = hexdigits[ch & 0x0000000F];
+	    continue;
+        }
+#else
+	/* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
+	else if (ch >= 0xD800 && ch < 0xDC00) {
+	    Py_UNICODE ch2;
+	    Py_UCS4 ucs;
+
+	    ch2 = *s++;
+	    size--;
+	    if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
+		ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
+		*p++ = '\\';
+		*p++ = 'U';
+		*p++ = hexdigits[(ucs >> 28) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 24) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 20) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 16) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 12) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 8) & 0x0000000F];
+		*p++ = hexdigits[(ucs >> 4) & 0x0000000F];
+		*p++ = hexdigits[ucs & 0x0000000F];
+		continue;
+	    }
+	    /* Fall through: isolated surrogates are copied as-is */
+	    s--;
+	    size++;
+	}
+#endif
+
+        /* Map 16-bit characters to '\uxxxx' */
+        if (ch >= 256) {
+            *p++ = '\\';
+            *p++ = 'u';
+            *p++ = hexdigits[(ch >> 12) & 0x000F];
+            *p++ = hexdigits[(ch >> 8) & 0x000F];
+            *p++ = hexdigits[(ch >> 4) & 0x000F];
+            *p++ = hexdigits[ch & 0x000F];
+        }
+
+        /* Map special whitespace to '\t', \n', '\r' */
+        else if (ch == '\t') {
             *p++ = '\\';
             *p++ = 't';
         }
@@ -7571,79 +7594,16 @@ PyObject *unicode_repr(PyObject *unicode)
         }
 
         /* Map non-printable US ASCII to '\xhh' */
-        else if (ch < ' ' || ch == 0x7F) {
+        else if (ch < ' ' || ch >= 0x7F) {
             *p++ = '\\';
             *p++ = 'x';
             *p++ = hexdigits[(ch >> 4) & 0x000F];
             *p++ = hexdigits[ch & 0x000F];
         }
 
-        /* Copy ASCII characters as-is */
+        /* Copy everything else as-is */
-        else if (ch < 0x7F) {
+        else
-            *p++ = ch;
+            *p++ = (char) ch;
-        }
-
-	/* Non-ASCII characters */
-        else {
-            Py_UCS4 ucs = ch;
-
-#ifndef Py_UNICODE_WIDE
-            Py_UNICODE ch2 = 0;
-            /* Get code point from surrogate pair */
-            if (size > 0) {
-                ch2 = *s;
-                if (ch >= 0xD800 && ch < 0xDC00 && ch2 >= 0xDC00
-                            && ch2 <= 0xDFFF) {
-                    ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) 
-                            + 0x00010000;
-                    s++; 
-                    size--;
-                }
-            }
-#endif
-            /* Map Unicode whitespace and control characters 
-               (categories Z* and C* except ASCII space)
-            */
-            if (!Py_UNICODE_ISPRINTABLE(ucs)) {
-                /* Map 8-bit characters to '\xhh' */
-                if (ucs <= 0xff) {
-                    *p++ = '\\';
-                    *p++ = 'x';
-                    *p++ = hexdigits[(ch >> 4) & 0x000F];
-                    *p++ = hexdigits[ch & 0x000F];
-                }
-                /* Map 21-bit characters to '\U00xxxxxx' */
-                else if (ucs >= 0x10000) {
-                    *p++ = '\\';
-                    *p++ = 'U';
-                    *p++ = hexdigits[(ucs >> 28) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 24) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 20) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 16) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 12) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 8) & 0x0000000F];
-                    *p++ = hexdigits[(ucs >> 4) & 0x0000000F];
-                    *p++ = hexdigits[ucs & 0x0000000F];
-                }
-                /* Map 16-bit characters to '\uxxxx' */
-                else {
-                    *p++ = '\\';
-                    *p++ = 'u';
-                    *p++ = hexdigits[(ucs >> 12) & 0x000F];
-                    *p++ = hexdigits[(ucs >> 8) & 0x000F];
-                    *p++ = hexdigits[(ucs >> 4) & 0x000F];
-                    *p++ = hexdigits[ucs & 0x000F];
-                }
-            }
-            /* Copy characters as-is */
-            else {
-                *p++ = ch;
-#ifndef Py_UNICODE_WIDE
-                if (ucs >= 0x10000)
-                    *p++ = ch2;
-#endif
-            }
-        }
     }
     /* Add quote */
     *p++ = PyUnicode_AS_UNICODE(repr)[0];
@@ -8308,7 +8268,6 @@ static PyMethodDef unicode_methods[] = {
     {"isalpha", (PyCFunction) unicode_isalpha, METH_NOARGS, isalpha__doc__},
     {"isalnum", (PyCFunction) unicode_isalnum, METH_NOARGS, isalnum__doc__},
     {"isidentifier", (PyCFunction) unicode_isidentifier, METH_NOARGS, isidentifier__doc__},
-    {"isprintable", (PyCFunction) unicode_isprintable, METH_NOARGS, isprintable__doc__},
     {"zfill", (PyCFunction) unicode_zfill, METH_VARARGS, zfill__doc__},
     {"format", (PyCFunction) do_string_format, METH_VARARGS | METH_KEYWORDS, format__doc__},
     {"__format__", (PyCFunction) unicode__format__, METH_VARARGS, p_format__doc__},
@@ -8894,7 +8853,6 @@ PyObject *PyUnicode_Format(PyObject *format,
 
 	    case 's':
 	    case 'r':
-	    case 'a':
 		if (PyUnicode_Check(v) && c == 's') {
 		    temp = v;
 		    Py_INCREF(temp);
@@ -8914,22 +8872,6 @@ PyObject *PyUnicode_Format(PyObject *format,
 					"%s argument has non-string str()");
 			goto onError;
 		    }
-		    if (c == 'a') {
-		        PyObject *ascii = PyUnicode_EncodeASCII(
-				PyUnicode_AS_UNICODE(temp),
-				PyUnicode_GET_SIZE(temp),
-				"backslashreplace");
-
-			Py_DECREF(temp);
-			if (ascii == NULL) 
-			    goto onError;
-
-			temp = PyUnicode_FromEncodedObject(ascii, 
-				    "ASCII", NULL);
-			Py_DECREF(ascii);
-			if (temp == NULL)
-			    goto onError;
-		    }
 		}
 		pbuf = PyUnicode_AS_UNICODE(temp);
 		len = PyUnicode_GET_SIZE(temp);

PC/VC6/rt.bat

@@ -31,11 +31,11 @@
 @if "%_qmode%"=="yes" goto Qmode
 @echo Deleting .pyc/.pyo files ...
 @%_exe% rmpyc.py
-%_exe% %_dashO% -E ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+%_exe% %_dashO% -E -tt ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
 @echo About to run again without deleting .pyc/.pyo first:
 @pause
 :Qmode
-%_exe% %_dashO% -E ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+%_exe% %_dashO% -E -tt ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
 @set _exe=
 @set _qmode=
 @set _dashO=

PC/VS7.1/rt.bat

@@ -34,7 +34,7 @@ if "%1"=="-O" (set dashO=-O)     & shift & goto CheckOpts
 if "%1"=="-q" (set qmode=yes)    & shift & goto CheckOpts
 if "%1"=="-d" (set exe=python_d) & shift & goto CheckOpts
 
-set cmd=%exe% %dashO% -E ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+set cmd=%exe% %dashO% -E -tt ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
 if defined qmode goto Qmode
 
 echo Deleting .pyc/.pyo files ...

PC/VS8.0/rt.bat

@@ -34,7 +34,7 @@ if "%1"=="-O" (set dashO=-O)     & shift & goto CheckOpts
 if "%1"=="-q" (set qmode=yes)    & shift & goto CheckOpts
 if "%1"=="-d" (set exe=python_d) & shift & goto CheckOpts
 
-set cmd=%exe% %dashO% -E ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+set cmd=%exe% %dashO% -E -tt ../../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
 if defined qmode goto Qmode
 
 echo Deleting .pyc/.pyo files ...

PC/os2emx/Makefile

@@ -666,7 +666,7 @@ _ssl$(MODULE.EXT): $(OUT)_ssl$O $(OUT)_ssl_m.def $(PYTHON.IMPLIB)
 # the test target
 test:
 	-find ../../Lib -name "*.py[co]" -exec rm {} ";"
-	-./python -E ../../lib/test/regrtest.py -l -u "network"
+	-./python -E -tt ../../lib/test/regrtest.py -l -u "network"
-	./python -E ../../lib/test/regrtest.py -l -u "network"
+	./python -E -tt ../../lib/test/regrtest.py -l -u "network"
 
 -include $(OUTBASE)python.dep

PC/os2emx/python26.def

@@ -49,6 +49,7 @@ EXPORTS
   "PyParser_Delete"
 
 ; From python26_s.lib(parsetok)
+  "Py_TabcheckFlag"
   "PyParser_ParseString"
   "PyParser_ParseStringFlagsFilename"
   "PyParser_ParseFile"

PC/os2vacpp/python.def

@@ -61,6 +61,7 @@ EXPORTS
                Py_InteractiveFlag
                Py_NoSiteFlag
                Py_OptimizeFlag
+               Py_TabcheckFlag
                Py_UseClassExceptionsFlag
                Py_VerboseFlag
                _PyImport_Filetab

PCbuild/rt.bat

@@ -40,7 +40,7 @@ if "%1"=="-x64" (set prefix=amd64) & (set tcltk=tcltk64) & shift & goto CheckOpt
 
 PATH %PATH%;..\..\%tcltk%\bin
 set exe=%prefix%\python%suffix%
-set cmd=%exe% %dashO% -E ../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
+set cmd=%exe% %dashO% -E -tt ../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9
 if defined qmode goto Qmode
 
 echo Deleting .pyc/.pyo files ...

Parser/parsetok.c

@@ -10,6 +10,8 @@
 #include "errcode.h"
 #include "graminit.h"
 
+int Py_TabcheckFlag;
+
 
 /* Forward */
 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
@@ -55,6 +57,9 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
 	}
 
         tok->filename = filename ? filename : "<string>";
+	if (Py_TabcheckFlag >= 3)
+		tok->alterror = 0;
+
 	return parsetok(tok, g, start, err_ret, flags);
 }
 
@@ -92,6 +97,9 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
 		return NULL;
 	}
 	tok->filename = filename;
+	if (Py_TabcheckFlag >= 3)
+		tok->alterror = 0;
+
 	return parsetok(tok, g, start, err_ret, flags);
 }
 

Python/bltinmodule.c

@@ -265,38 +265,6 @@ PyDoc_STRVAR(any_doc,
 \n\
 Return True if bool(x) is True for any x in the iterable.");
 
-static PyObject *
-builtin_ascii(PyObject *self, PyObject *v)
-{
-	PyObject *repr, *bytes, *ascii;
-	repr = PyObject_Repr(v);
-	if (!repr)
-	    return NULL;
-
-        bytes = PyUnicode_EncodeASCII(
-		PyUnicode_AS_UNICODE(repr),
-		PyUnicode_GET_SIZE(repr),
-		"backslashreplace");
-
-	Py_DECREF(repr);
-	if (bytes == NULL) 
-	    return NULL;
-
-	ascii = PyUnicode_FromEncodedObject(bytes, 
-		    "ASCII", NULL);
-	Py_DECREF(bytes);
-	if (ascii == NULL)
-	    return NULL;
-
-	return ascii;
-}
-
-PyDoc_STRVAR(ascii_doc,
-"ascii(object) -> string\n\
-\n\
-Return the canonical string representation of the object as repr(),\n\
-but non-ASCII characters in the string are hex-escaped");
-
 
 static PyObject *
 builtin_bin(PyObject *self, PyObject *v)
@@ -2220,7 +2188,6 @@ static PyMethodDef builtin_methods[] = {
  	{"abs",		builtin_abs,        METH_O, abs_doc},
  	{"all",		builtin_all,        METH_O, all_doc},
  	{"any",		builtin_any,        METH_O, any_doc},
- 	{"ascii",	builtin_ascii,      METH_O, ascii_doc},
 	{"bin",		builtin_bin,	    METH_O, bin_doc},
  	{"chr",		builtin_chr,        METH_VARARGS, chr_doc},
  	{"cmp",		builtin_cmp,        METH_VARARGS, cmp_doc},

Python/sysmodule.c

@@ -1100,6 +1100,7 @@ static PyStructSequence_Field flags_fields[] = {
 	{"no_user_site",	"-s"},
 	{"no_site",		"-S"},
 	{"ignore_environment",	"-E"},
+	{"tabcheck",		"-t or -tt"},
 	{"verbose",		"-v"},
 #ifdef RISCOS
 	{"riscos_wimp",		"???"},
@@ -1115,9 +1116,9 @@ static PyStructSequence_Desc flags_desc = {
 	flags__doc__,	/* doc */
 	flags_fields,	/* fields */
 #ifdef RISCOS
-	11
+	12
 #else
-	10
+	11
 #endif
 };
 
@@ -1143,6 +1144,7 @@ make_flags(void)
 	SetFlag(Py_NoUserSiteDirectory);
 	SetFlag(Py_NoSiteFlag);
 	SetFlag(Py_IgnoreEnvironmentFlag);
+	SetFlag(Py_TabcheckFlag);
 	SetFlag(Py_VerboseFlag);
 #ifdef RISCOS
 	SetFlag(Py_RISCOSWimpFlag);

Tools/unicode/makeunicodedata.py

@@ -60,7 +60,6 @@ TITLE_MASK = 0x40
 UPPER_MASK = 0x80
 XID_START_MASK = 0x100
 XID_CONTINUE_MASK = 0x200
-NONPRINTABLE_MASK = 0x400
 
 def maketables(trace=0):
 
@@ -72,7 +71,7 @@ def maketables(trace=0):
                           EASTASIAN_WIDTH % version,
                           DERIVED_CORE_PROPERTIES % version)
 
-    print(len(list(filter(None, unicode.table))), "characters")
+    print(len(filter(None, unicode.table)), "characters")
 
     for version in old_versions:
         print("--- Reading", UNICODE_DATA % ("-"+version), "...")
@@ -80,7 +79,7 @@ def maketables(trace=0):
                                   COMPOSITION_EXCLUSIONS % ("-"+version),
                                   EASTASIAN_WIDTH % ("-"+version),
                                   DERIVED_CORE_PROPERTIES % ("-"+version))
-        print(len(list(filter(None, old_unicode.table))), "characters")
+        print(len(filter(None, old_unicode.table)), "characters")
         merge_old_version(version, unicode, old_unicode)
 
     makeunicodename(unicode, trace)
@@ -372,10 +371,6 @@ def makeunicodetype(unicode, trace):
                 flags |= TITLE_MASK
             if category == "Lu":
                 flags |= UPPER_MASK
-            if category[0] == "C":
-                flags |= NONPRINTABLE_MASK
-            if category[0] == "Z" and char != " ":
-                flags |= NONPRINTABLE_MASK
             if "XID_Start" in properties:
                 flags |= XID_START_MASK
             if "XID_Continue" in properties:
@@ -470,7 +465,7 @@ def makeunicodename(unicode, trace):
             if name and name[0] != "<":
                 names[char] = name + chr(0)
 
-    print(len(list(n for n in names if n is not None)), "distinct names")
+    print(len(n for n in names if n is not None), "distinct names")
 
     # collect unique words from names (note that we differ between
     # words inside a sentence, and words ending a sentence.  the