Merge heads

Doc/howto/logging-cookbook.rst

@@ -1680,7 +1680,7 @@ as in the following complete example::
 
     def main():
         logging.basicConfig(level=logging.INFO, format='%(message)s')
-        logging.info(_('message 1', set_value=set([1, 2, 3]), snowman='\u2603'))
+        logging.info(_('message 1', set_value={1, 2, 3}, snowman='\u2603'))
 
     if __name__ == '__main__':
         main()

Doc/library/functools.rst

@@ -21,8 +21,8 @@ The :mod:`functools` module defines the following functions:
 
 .. function:: cmp_to_key(func)
 
-   Transform an old-style comparison function to a key function.  Used with
+   Transform an old-style comparison function to a :term:`key function`.  Used
-   tools that accept key functions (such as :func:`sorted`, :func:`min`,
+   with tools that accept key functions (such as :func:`sorted`, :func:`min`,
    :func:`max`, :func:`heapq.nlargest`, :func:`heapq.nsmallest`,
    :func:`itertools.groupby`).  This function is primarily used as a transition
    tool for programs being converted from Python 2 which supported the use of
@@ -31,13 +31,14 @@ The :mod:`functools` module defines the following functions:
    A comparison function is any callable that accept two arguments, compares them,
    and returns a negative number for less-than, zero for equality, or a positive
    number for greater-than.  A key function is a callable that accepts one
-   argument and returns another value indicating the position in the desired
+   argument and returns another value to be used as the sort key.
-   collation sequence.
 
    Example::
 
        sorted(iterable, key=cmp_to_key(locale.strcoll))  # locale-aware sort order
 
+   For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`.
+
    .. versionadded:: 3.2
 
 

Doc/library/json.rst

@@ -567,6 +567,7 @@ the last name-value pair for a given name::
 The *object_pairs_hook* parameter can be used to alter this behavior.
 
 .. highlight:: bash
+.. module:: json.tool
 
 .. _json-commandline:
 
@@ -586,6 +587,10 @@ specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively::
     $ echo '{1.2:3.4}' | python -m json.tool
     Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
 
+.. versionchanged:: 3.5
+   The output is now in the same order as the input. Use the
+   :option:`--sort-keys` option to sort the output of dictionaries
+   alphabetically by key.
 
 Command line options
 ^^^^^^^^^^^^^^^^^^^^
@@ -613,6 +618,12 @@ Command line options
    Write the output of the *infile* to the given *outfile*. Otherwise, write it
    to :attr:`sys.stdout`.
 
+.. cmdoption:: --sort-keys
+
+   Sort the output of dictionaries alphabetically by key.
+
+   .. versionadded:: 3.5
+
 .. cmdoption:: -h, --help
 
    Show the help message.

Doc/library/pickle.rst

@@ -859,7 +859,7 @@ For the simplest code, use the :func:`dump` and :func:`load` functions. ::
    data = {
        'a': [1, 2.0, 3, 4+6j],
        'b': ("character string", b"byte string"),
-       'c': set([None, True, False])
+       'c': {None, True, False}
    }
 
    with open('data.pickle', 'wb') as f:

Doc/tutorial/stdlib2.rst

@@ -18,7 +18,7 @@ abbreviated displays of large or deeply nested containers::
 
    >>> import reprlib
    >>> reprlib.repr(set('supercalifragilisticexpialidocious'))
-   "set(['a', 'c', 'd', 'e', 'f', 'g', ...])"
+   "{'a', 'c', 'd', 'e', 'f', 'g', ...}"
 
 The :mod:`pprint` module offers more sophisticated control over printing both
 built-in and user defined objects in a way that is readable by the interpreter.

Doc/whatsnew/3.5.rst

@@ -211,6 +211,14 @@ ipaddress
   network objects from existing addresses.  (Contributed by Peter Moody
   and Antoine Pitrou in :issue:`16531`.)
 
+json
+----
+
+* The output of :mod:`json.tool` command line interface is now in the same
+  order as the input. Use the :option:`--sort-keys` option to sort the output
+  of dictionaries alphabetically by key.  (Contributed by Berker Peksag in
+  :issue:`21650`.)
+
 os
 --
 

Lib/_strptime.py

@@ -167,9 +167,9 @@ class LocaleTime(object):
             time.tzset()
         except AttributeError:
             pass
-        no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()])
+        no_saving = frozenset({"utc", "gmt", time.tzname[0].lower()})
         if time.daylight:
-            has_saving = frozenset([time.tzname[1].lower()])
+            has_saving = frozenset({time.tzname[1].lower()})
         else:
             has_saving = frozenset()
         self.timezone = (no_saving, has_saving)

Lib/asyncore.py

@@ -57,8 +57,8 @@ from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
      ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
      errorcode
 
-_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
-                           EBADF))
+                           EBADF})
 
 try:
     socket_map
@@ -220,7 +220,7 @@ class dispatcher:
     connecting = False
     closing = False
     addr = None
-    ignore_log_types = frozenset(['warning'])
+    ignore_log_types = frozenset({'warning'})
 
     def __init__(self, sock=None, map=None):
         if map is None:

Lib/ipaddress.py

@@ -1088,7 +1088,7 @@ class _BaseV4:
     _DECIMAL_DIGITS = frozenset('0123456789')
 
     # the valid octets for host and netmasks. only useful for IPv4.
-    _valid_mask_octets = frozenset((255, 254, 252, 248, 240, 224, 192, 128, 0))
+    _valid_mask_octets = frozenset({255, 254, 252, 248, 240, 224, 192, 128, 0})
 
     _max_prefixlen = IPV4LENGTH
     # There are only a handful of valid v4 netmasks, so we cache them all

Lib/json/tool.py

@@ -11,6 +11,7 @@ Usage::
 
 """
 import argparse
+import collections
 import json
 import sys
 
@@ -24,17 +25,24 @@ def main():
                         help='a JSON file to be validated or pretty-printed')
     parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
                         help='write the output of infile to outfile')
+    parser.add_argument('--sort-keys', action='store_true', default=False,
+                        help='sort the output of dictionaries alphabetically by key')
     options = parser.parse_args()
 
     infile = options.infile or sys.stdin
     outfile = options.outfile or sys.stdout
+    sort_keys = options.sort_keys
     with infile:
         try:
-            obj = json.load(infile)
+            if sort_keys:
+                obj = json.load(infile)
+            else:
+                obj = json.load(infile,
+                                object_pairs_hook=collections.OrderedDict)
         except ValueError as e:
             raise SystemExit(e)
     with outfile:
-        json.dump(obj, outfile, sort_keys=True, indent=4)
+        json.dump(obj, outfile, sort_keys=sort_keys, indent=4)
         outfile.write('\n')
 
 

Lib/mailbox.py

@@ -1230,8 +1230,8 @@ class MH(Mailbox):
 class Babyl(_singlefileMailbox):
     """An Rmail-style Babyl mailbox."""
 
-    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+    _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered',
-                                 'forwarded', 'edited', 'resent'))
+                                 'forwarded', 'edited', 'resent'})
 
     def __init__(self, path, factory=None, create=True):
         """Initialize a Babyl mailbox."""

Lib/reprlib.py

@@ -87,12 +87,16 @@ class Repr:
         return self._repr_iterable(x, level, header, '])', self.maxarray)
 
     def repr_set(self, x, level):
+        if not x:
+            return 'set()'
         x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, 'set([', '])', self.maxset)
+        return self._repr_iterable(x, level, '{', '}', self.maxset)
 
     def repr_frozenset(self, x, level):
+        if not x:
+            return 'frozenset()'
         x = _possibly_sorted(x)
-        return self._repr_iterable(x, level, 'frozenset([', '])',
+        return self._repr_iterable(x, level, 'frozenset({', '})',
                                    self.maxfrozenset)
 
     def repr_deque(self, x, level):

Lib/sre_compile.py

@@ -21,10 +21,10 @@ if _sre.CODESIZE == 2:
 else:
     MAXCODE = 0xFFFFFFFF
 
-_LITERAL_CODES = set([LITERAL, NOT_LITERAL])
+_LITERAL_CODES = {LITERAL, NOT_LITERAL}
-_REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT])
+_REPEATING_CODES = {REPEAT, MIN_REPEAT, MAX_REPEAT}
-_SUCCESS_CODES = set([SUCCESS, FAILURE])
+_SUCCESS_CODES = {SUCCESS, FAILURE}
-_ASSERT_CODES = set([ASSERT, ASSERT_NOT])
+_ASSERT_CODES = {ASSERT, ASSERT_NOT}
 
 def _compile(code, pattern, flags):
     # internal: compile a (sub)pattern

Lib/sre_parse.py

@@ -24,8 +24,8 @@ HEXDIGITS = frozenset("0123456789abcdefABCDEF")
 
 WHITESPACE = frozenset(" \t\n\r\v\f")
 
-_REPEATCODES = frozenset((MIN_REPEAT, MAX_REPEAT))
+_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT})
-_UNITCODES = frozenset((ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY))
+_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY})
 
 ESCAPES = {
     r"\a": (LITERAL, ord("\a")),

Lib/statistics.py

@@ -150,7 +150,7 @@ def _sum(data, start=0):
     # We fail as soon as we reach a value that is not an int or the type of
     # the first value which is not an int. E.g. _sum([int, int, float, int])
     # is okay, but sum([int, int, float, Fraction]) is not.
-    allowed_types = set([int, type(start)])
+    allowed_types = {int, type(start)}
     n, d = _exact_ratio(start)
     partials = {d: n}  # map {denominator: sum of numerators}
     # Micro-optimizations.
@@ -168,7 +168,7 @@ def _sum(data, start=0):
         assert allowed_types.pop() is int
         T = int
     else:
-        T = (allowed_types - set([int])).pop()
+        T = (allowed_types - {int}).pop()
     if None in partials:
         assert issubclass(T, (float, Decimal))
         assert not math.isfinite(partials[None])

Lib/test/test_json/test_tool.py

@@ -6,6 +6,7 @@ import subprocess
 from test import support
 from test.script_helper import assert_python_ok
 
+
 class TestTool(unittest.TestCase):
     data = """
 
@@ -15,7 +16,7 @@ class TestTool(unittest.TestCase):
             :"yes"}  ]
            """
 
-    expect = textwrap.dedent("""\
+    expect_without_sort_keys = textwrap.dedent("""\
     [
         [
             "blorpie"
@@ -37,6 +38,28 @@ class TestTool(unittest.TestCase):
     ]
     """)
 
+    expect = textwrap.dedent("""\
+    [
+        [
+            "blorpie"
+        ],
+        [
+            "whoops"
+        ],
+        [],
+        "d-shtaeou",
+        "d-nthiouh",
+        "i-vhbjkhnth",
+        {
+            "nifty": 87
+        },
+        {
+            "morefield": false,
+            "field": "yes"
+        }
+    ]
+    """)
+
     def test_stdin_stdout(self):
         with subprocess.Popen(
                 (sys.executable, '-m', 'json.tool'),
@@ -75,3 +98,11 @@ class TestTool(unittest.TestCase):
         self.assertEqual(rc, 0)
         self.assertTrue(out.startswith(b'usage: '))
         self.assertEqual(err, b'')
+
+    def test_sort_keys_flag(self):
+        infile = self._create_infile()
+        rc, out, err = assert_python_ok('-m', 'json.tool', '--sort-keys', infile)
+        self.assertEqual(rc, 0)
+        self.assertEqual(out.splitlines(),
+                         self.expect_without_sort_keys.encode().splitlines())
+        self.assertEqual(err, b'')

Lib/test/test_reprlib.py

@@ -10,7 +10,7 @@ import importlib
 import importlib.util
 import unittest
 
-from test.support import run_unittest, create_empty_file, verbose
+from test.support import create_empty_file, verbose
 from reprlib import repr as r # Don't shadow builtin repr
 from reprlib import Repr
 from reprlib import recursive_repr
@@ -70,18 +70,18 @@ class ReprTests(unittest.TestCase):
         eq(r([1, 2, 3, 4, 5, 6, 7]), "[1, 2, 3, 4, 5, 6, ...]")
 
         # Sets give up after 6 as well
-        eq(r(set([])), "set([])")
+        eq(r(set([])), "set()")
-        eq(r(set([1])), "set([1])")
+        eq(r(set([1])), "{1}")
-        eq(r(set([1, 2, 3])), "set([1, 2, 3])")
+        eq(r(set([1, 2, 3])), "{1, 2, 3}")
-        eq(r(set([1, 2, 3, 4, 5, 6])), "set([1, 2, 3, 4, 5, 6])")
+        eq(r(set([1, 2, 3, 4, 5, 6])), "{1, 2, 3, 4, 5, 6}")
-        eq(r(set([1, 2, 3, 4, 5, 6, 7])), "set([1, 2, 3, 4, 5, 6, ...])")
+        eq(r(set([1, 2, 3, 4, 5, 6, 7])), "{1, 2, 3, 4, 5, 6, ...}")
 
         # Frozensets give up after 6 as well
-        eq(r(frozenset([])), "frozenset([])")
+        eq(r(frozenset([])), "frozenset()")
-        eq(r(frozenset([1])), "frozenset([1])")
+        eq(r(frozenset([1])), "frozenset({1})")
-        eq(r(frozenset([1, 2, 3])), "frozenset([1, 2, 3])")
+        eq(r(frozenset([1, 2, 3])), "frozenset({1, 2, 3})")
-        eq(r(frozenset([1, 2, 3, 4, 5, 6])), "frozenset([1, 2, 3, 4, 5, 6])")
+        eq(r(frozenset([1, 2, 3, 4, 5, 6])), "frozenset({1, 2, 3, 4, 5, 6})")
-        eq(r(frozenset([1, 2, 3, 4, 5, 6, 7])), "frozenset([1, 2, 3, 4, 5, 6, ...])")
+        eq(r(frozenset([1, 2, 3, 4, 5, 6, 7])), "frozenset({1, 2, 3, 4, 5, 6, ...})")
 
         # collections.deque after 6
         eq(r(deque([1, 2, 3, 4, 5, 6, 7])), "deque([1, 2, 3, 4, 5, 6, ...])")
@@ -103,6 +103,20 @@ class ReprTests(unittest.TestCase):
         eq(r(array('i', [1, 2, 3, 4, 5, 6])),
                    "array('i', [1, 2, 3, 4, 5, ...])")
 
+    def test_set_literal(self):
+        eq = self.assertEqual
+        eq(r({1}), "{1}")
+        eq(r({1, 2, 3}), "{1, 2, 3}")
+        eq(r({1, 2, 3, 4, 5, 6}), "{1, 2, 3, 4, 5, 6}")
+        eq(r({1, 2, 3, 4, 5, 6, 7}), "{1, 2, 3, 4, 5, 6, ...}")
+
+    def test_frozenset(self):
+        eq = self.assertEqual
+        eq(r(frozenset({1})), "frozenset({1})")
+        eq(r(frozenset({1, 2, 3})), "frozenset({1, 2, 3})")
+        eq(r(frozenset({1, 2, 3, 4, 5, 6})), "frozenset({1, 2, 3, 4, 5, 6})")
+        eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})")
+
     def test_numbers(self):
         eq = self.assertEqual
         eq(r(123), repr(123))
@@ -373,11 +387,5 @@ class TestRecursiveRepr(unittest.TestCase):
         m.append(m)
         self.assertEqual(repr(m), '<a, b, c, d, e, +++, x, +++>')
 
-def test_main():
-    run_unittest(ReprTests)
-    run_unittest(LongReprTest)
-    run_unittest(TestRecursiveRepr)
-
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

Misc/NEWS

@@ -183,9 +183,14 @@ Core and Builtins
 Library
 -------
 
+- Issue #21650: Add an `--sort-keys` option to json.tool CLI.
+
 - Issues #814253, #9179: Group references and conditional group references now
   work in lookbehind assertions in regular expressions.
 
+- Issue #22824:  Updated reprlib output format for sets to use set literals.
+  Patch contributed by Berker Peksag.
+
 - Issue #22406: Fixed the uu_codec codec incorrectly ported to 3.x.
   Based on patch by Martin Panter.
 

Modules/clinic/fcntlmodule.c.h

@@ -0,0 +1,188 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+PyDoc_STRVAR(fcntl_fcntl__doc__,
+"fcntl($module, fd, code, arg=None, /)\n"
+"--\n"
+"\n"
+"Perform the operation `code` on file descriptor fd.\n"
+"\n"
+"The values used for `code` are operating system dependent, and are available\n"
+"as constants in the fcntl module, using the same names as used in\n"
+"the relevant C header files.  The argument arg is optional, and\n"
+"defaults to 0; it may be an int or a string.  If arg is given as a string,\n"
+"the return value of fcntl is a string of that length, containing the\n"
+"resulting value put in the arg buffer by the operating system.  The length\n"
+"of the arg string is not allowed to exceed 1024 bytes.  If the arg given\n"
+"is an integer or if none is specified, the result value is an integer\n"
+"corresponding to the return value of the fcntl call in the C code.");
+
+#define FCNTL_FCNTL_METHODDEF    \
+    {"fcntl", (PyCFunction)fcntl_fcntl, METH_VARARGS, fcntl_fcntl__doc__},
+
+static PyObject *
+fcntl_fcntl_impl(PyModuleDef *module, int fd, int code, PyObject *arg);
+
+static PyObject *
+fcntl_fcntl(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    int fd;
+    int code;
+    PyObject *arg = NULL;
+
+    if (!PyArg_ParseTuple(args,
+        "O&i|O:fcntl",
+        conv_descriptor, &fd, &code, &arg))
+        goto exit;
+    return_value = fcntl_fcntl_impl(module, fd, code, arg);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(fcntl_ioctl__doc__,
+"ioctl($module, fd, op, arg=None, mutate_flag=True, /)\n"
+"--\n"
+"\n"
+"Perform the operation op on file descriptor fd.\n"
+"\n"
+"The values used for op are operating system dependent, and are available as\n"
+"constants in the fcntl or termios library modules, using the same names as\n"
+"used in the relevant C header files.\n"
+"\n"
+"The argument `arg` is optional, and defaults to 0; it may be an int or a\n"
+"buffer containing character data (most likely a string or an array).\n"
+"\n"
+"If the argument is a mutable buffer (such as an array) and if the\n"
+"mutate_flag argument (which is only allowed in this case) is true then the\n"
+"buffer is (in effect) passed to the operating system and changes made by\n"
+"the OS will be reflected in the contents of the buffer after the call has\n"
+"returned.  The return value is the integer returned by the ioctl system\n"
+"call.\n"
+"\n"
+"If the argument is a mutable buffer and the mutable_flag argument is not\n"
+"passed or is false, the behavior is as if a string had been passed.  This\n"
+"behavior will change in future releases of Python.\n"
+"\n"
+"If the argument is an immutable buffer (most likely a string) then a copy\n"
+"of the buffer is passed to the operating system and the return value is a\n"
+"string of the same length containing whatever the operating system put in\n"
+"the buffer.  The length of the arg buffer in this case is not allowed to\n"
+"exceed 1024 bytes.\n"
+"\n"
+"If the arg given is an integer or if none is specified, the result value is\n"
+"an integer corresponding to the return value of the ioctl call in the C\n"
+"code.");
+
+#define FCNTL_IOCTL_METHODDEF    \
+    {"ioctl", (PyCFunction)fcntl_ioctl, METH_VARARGS, fcntl_ioctl__doc__},
+
+static PyObject *
+fcntl_ioctl_impl(PyModuleDef *module, int fd, unsigned int code, PyObject *ob_arg, int mutate_arg);
+
+static PyObject *
+fcntl_ioctl(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    int fd;
+    unsigned int code;
+    PyObject *ob_arg = NULL;
+    int mutate_arg = 1;
+
+    if (!PyArg_ParseTuple(args,
+        "O&I|Op:ioctl",
+        conv_descriptor, &fd, &code, &ob_arg, &mutate_arg))
+        goto exit;
+    return_value = fcntl_ioctl_impl(module, fd, code, ob_arg, mutate_arg);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(fcntl_flock__doc__,
+"flock($module, fd, code, /)\n"
+"--\n"
+"\n"
+"Perform the lock operation op on file descriptor fd.\n"
+"\n"
+"See the Unix manual page for flock(2) for details (On some systems, this\n"
+"function is emulated using fcntl()).");
+
+#define FCNTL_FLOCK_METHODDEF    \
+    {"flock", (PyCFunction)fcntl_flock, METH_VARARGS, fcntl_flock__doc__},
+
+static PyObject *
+fcntl_flock_impl(PyModuleDef *module, int fd, int code);
+
+static PyObject *
+fcntl_flock(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    int fd;
+    int code;
+
+    if (!PyArg_ParseTuple(args,
+        "O&i:flock",
+        conv_descriptor, &fd, &code))
+        goto exit;
+    return_value = fcntl_flock_impl(module, fd, code);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(fcntl_lockf__doc__,
+"lockf($module, fd, code, lenobj=None, startobj=None, whence=0, /)\n"
+"--\n"
+"\n"
+"A wrapper around the fcntl() locking calls.\n"
+"\n"
+"fd is the file descriptor of the file to lock or unlock, and operation is one\n"
+"of the following values:\n"
+"\n"
+"    LOCK_UN - unlock\n"
+"    LOCK_SH - acquire a shared lock\n"
+"    LOCK_EX - acquire an exclusive lock\n"
+"\n"
+"When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n"
+"LOCK_NB to avoid blocking on lock acquisition.  If LOCK_NB is used and the\n"
+"lock cannot be acquired, an IOError will be raised and the exception will\n"
+"have an errno attribute set to EACCES or EAGAIN (depending on the operating\n"
+"system -- for portability, check for either value).\n"
+"\n"
+"length is the number of bytes to lock, with the default meaning to lock to\n"
+"EOF.  start is the byte offset, relative to whence, to that the lock\n"
+"starts.  whence is as with fileobj.seek(), specifically:\n"
+"\n"
+"    0 - relative to the start of the file (SEEK_SET)\n"
+"    1 - relative to the current buffer position (SEEK_CUR)\n"
+"    2 - relative to the end of the file (SEEK_END)");
+
+#define FCNTL_LOCKF_METHODDEF    \
+    {"lockf", (PyCFunction)fcntl_lockf, METH_VARARGS, fcntl_lockf__doc__},
+
+static PyObject *
+fcntl_lockf_impl(PyModuleDef *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence);
+
+static PyObject *
+fcntl_lockf(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    int fd;
+    int code;
+    PyObject *lenobj = NULL;
+    PyObject *startobj = NULL;
+    int whence = 0;
+
+    if (!PyArg_ParseTuple(args,
+        "O&i|OOi:lockf",
+        conv_descriptor, &fd, &code, &lenobj, &startobj, &whence))
+        goto exit;
+    return_value = fcntl_lockf_impl(module, fd, code, lenobj, startobj, whence);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=84bdde73a92f7c61 input=a9049054013a1b77]*/

Modules/fcntlmodule.c

@@ -15,6 +15,12 @@
 #include <stropts.h>
 #endif
 
+/*[clinic input]
+output preset file
+module fcntl
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c7356fdb126a904a]*/
+
 static int
 conv_descriptor(PyObject *object, int *target)
 {
@@ -26,48 +32,72 @@ conv_descriptor(PyObject *object, int *target)
     return 1;
 }
 
+/* Must come after conv_descriptor definition. */
+#include "clinic/fcntlmodule.c.h"
 
-/* fcntl(fd, op, [arg]) */
+/*[clinic input]
+fcntl.fcntl
+
+    fd: object(type='int', converter='conv_descriptor')
+    code: int
+    arg: object = NULL
+    /
+
+Perform the operation `code` on file descriptor fd.
+
+The values used for `code` are operating system dependent, and are available
+as constants in the fcntl module, using the same names as used in
+the relevant C header files.  The argument arg is optional, and
+defaults to 0; it may be an int or a string.  If arg is given as a string,
+the return value of fcntl is a string of that length, containing the
+resulting value put in the arg buffer by the operating system.  The length
+of the arg string is not allowed to exceed 1024 bytes.  If the arg given
+is an integer or if none is specified, the result value is an integer
+corresponding to the return value of the fcntl call in the C code.
+[clinic start generated code]*/
 
 static PyObject *
-fcntl_fcntl(PyObject *self, PyObject *args)
+fcntl_fcntl_impl(PyModuleDef *module, int fd, int code, PyObject *arg)
+/*[clinic end generated code: output=afc5bfa74a03ef0d input=4850c13a41e86930]*/
 {
-    int fd;
+    int int_arg = 0;
-    int code;
-    long arg;
     int ret;
     char *str;
     Py_ssize_t len;
     char buf[1024];
 
-    if (PyArg_ParseTuple(args, "O&is#:fcntl",
+    if (arg != NULL) {
-                         conv_descriptor, &fd, &code, &str, &len)) {
+        int parse_result;
-        if ((size_t)len > sizeof buf) {
+
-            PyErr_SetString(PyExc_ValueError,
+        if (PyArg_Parse(arg, "s#", &str, &len)) {
-                            "fcntl string arg too long");
+            if ((size_t)len > sizeof buf) {
-            return NULL;
+                PyErr_SetString(PyExc_ValueError,
+                                "fcntl string arg too long");
+                return NULL;
+            }
+            memcpy(buf, str, len);
+            Py_BEGIN_ALLOW_THREADS
+            ret = fcntl(fd, code, buf);
+            Py_END_ALLOW_THREADS
+            if (ret < 0) {
+                PyErr_SetFromErrno(PyExc_IOError);
+                return NULL;
+            }
+            return PyBytes_FromStringAndSize(buf, len);
         }
-        memcpy(buf, str, len);
+
-        Py_BEGIN_ALLOW_THREADS
+        PyErr_Clear();
-        ret = fcntl(fd, code, buf);
+        parse_result = PyArg_Parse(arg,
-        Py_END_ALLOW_THREADS
+            "l;fcntl requires a file or file descriptor,"
-        if (ret < 0) {
+            " an integer and optionally a third integer or a string",
-            PyErr_SetFromErrno(PyExc_IOError);
+            &int_arg);
-            return NULL;
+        if (!parse_result) {
+          return NULL;
         }
-        return PyBytes_FromStringAndSize(buf, len);
     }
 
-    PyErr_Clear();
-    arg = 0;
-    if (!PyArg_ParseTuple(args,
-         "O&i|l;fcntl requires a file or file descriptor,"
-         " an integer and optionally a third integer or a string",
-                          conv_descriptor, &fd, &code, &arg)) {
-      return NULL;
-    }
     Py_BEGIN_ALLOW_THREADS
-    ret = fcntl(fd, code, arg);
+    ret = fcntl(fd, code, int_arg);
     Py_END_ALLOW_THREADS
     if (ret < 0) {
         PyErr_SetFromErrno(PyExc_IOError);
@@ -76,29 +106,53 @@ fcntl_fcntl(PyObject *self, PyObject *args)
     return PyLong_FromLong((long)ret);
 }
 
-PyDoc_STRVAR(fcntl_doc,
-"fcntl(fd, op, [arg])\n\
-\n\
-Perform the operation op on file descriptor fd.  The values used\n\
-for op are operating system dependent, and are available\n\
-as constants in the fcntl module, using the same names as used in\n\
-the relevant C header files.  The argument arg is optional, and\n\
-defaults to 0; it may be an int or a string.  If arg is given as a string,\n\
-the return value of fcntl is a string of that length, containing the\n\
-resulting value put in the arg buffer by the operating system.  The length\n\
-of the arg string is not allowed to exceed 1024 bytes.  If the arg given\n\
-is an integer or if none is specified, the result value is an integer\n\
-corresponding to the return value of the fcntl call in the C code.");
 
+/*[clinic input]
+fcntl.ioctl
 
-/* ioctl(fd, op, [arg]) */
+    fd: object(type='int', converter='conv_descriptor')
+    op as code: unsigned_int(bitwise=True)
+    arg as ob_arg: object = NULL
+    mutate_flag as mutate_arg: bool = True
+    /
+
+Perform the operation op on file descriptor fd.
+
+The values used for op are operating system dependent, and are available as
+constants in the fcntl or termios library modules, using the same names as
+used in the relevant C header files.
+
+The argument `arg` is optional, and defaults to 0; it may be an int or a
+buffer containing character data (most likely a string or an array).
+
+If the argument is a mutable buffer (such as an array) and if the
+mutate_flag argument (which is only allowed in this case) is true then the
+buffer is (in effect) passed to the operating system and changes made by
+the OS will be reflected in the contents of the buffer after the call has
+returned.  The return value is the integer returned by the ioctl system
+call.
+
+If the argument is a mutable buffer and the mutable_flag argument is not
+passed or is false, the behavior is as if a string had been passed.  This
+behavior will change in future releases of Python.
+
+If the argument is an immutable buffer (most likely a string) then a copy
+of the buffer is passed to the operating system and the return value is a
+string of the same length containing whatever the operating system put in
+the buffer.  The length of the arg buffer in this case is not allowed to
+exceed 1024 bytes.
+
+If the arg given is an integer or if none is specified, the result value is
+an integer corresponding to the return value of the ioctl call in the C
+code.
+[clinic start generated code]*/
 
 static PyObject *
-fcntl_ioctl(PyObject *self, PyObject *args)
+fcntl_ioctl_impl(PyModuleDef *module, int fd, unsigned int code, PyObject *ob_arg, int mutate_arg)
+/*[clinic end generated code: output=ad47738c118622bf input=a55a6ee8e494c449]*/
 {
 #define IOCTL_BUFSZ 1024
-    int fd;
+    /* We use the unsigned non-checked 'I'
-    /* In PyArg_ParseTuple below, we use the unsigned non-checked 'I'
        format for the 'code' parameter because Python turns 0x8000000
        into either a large positive number (PyLong or PyInt on 64-bit
        platforms) or a negative number on others (32-bit PyInt)
@@ -111,101 +165,98 @@ fcntl_ioctl(PyObject *self, PyObject *args)
        in their unsigned long ioctl codes this will break and need
        special casing based on the platform being built on.
      */
-    unsigned int code;
+    int arg = 0;
-    int arg;
     int ret;
     Py_buffer pstr;
     char *str;
     Py_ssize_t len;
-    int mutate_arg = 1;
     char buf[IOCTL_BUFSZ+1];  /* argument plus NUL byte */
 
-    if (PyArg_ParseTuple(args, "O&Iw*|i:ioctl",
+    if (ob_arg != NULL) {
-                         conv_descriptor, &fd, &code,
+        if (PyArg_Parse(ob_arg, "w*:ioctl", &pstr)) {
-                         &pstr, &mutate_arg)) {
+            char *arg;
-        char *arg;
+            str = pstr.buf;
-        str = pstr.buf;
+            len = pstr.len;
-        len = pstr.len;
 
-        if (mutate_arg) {
+            if (mutate_arg) {
-            if (len <= IOCTL_BUFSZ) {
+                if (len <= IOCTL_BUFSZ) {
-                memcpy(buf, str, len);
+                    memcpy(buf, str, len);
-                buf[len] = '\0';
+                    buf[len] = '\0';
-                arg = buf;
+                    arg = buf;
+                }
+                else {
+                    arg = str;
+                }
             }
             else {
-                arg = str;
+                if (len > IOCTL_BUFSZ) {
+                    PyBuffer_Release(&pstr);
+                    PyErr_SetString(PyExc_ValueError,
+                        "ioctl string arg too long");
+                    return NULL;
+                }
+                else {
+                    memcpy(buf, str, len);
+                    buf[len] = '\0';
+                    arg = buf;
+                }
+            }
+            if (buf == arg) {
+                Py_BEGIN_ALLOW_THREADS /* think array.resize() */
+                ret = ioctl(fd, code, arg);
+                Py_END_ALLOW_THREADS
+            }
+            else {
+                ret = ioctl(fd, code, arg);
+            }
+            if (mutate_arg && (len <= IOCTL_BUFSZ)) {
+                memcpy(str, buf, len);
+            }
+            PyBuffer_Release(&pstr); /* No further access to str below this point */
+            if (ret < 0) {
+                PyErr_SetFromErrno(PyExc_IOError);
+                return NULL;
+            }
+            if (mutate_arg) {
+                return PyLong_FromLong(ret);
+            }
+            else {
+                return PyBytes_FromStringAndSize(buf, len);
             }
         }
-        else {
+
+        PyErr_Clear();
+        if (PyArg_Parse(ob_arg, "s*:ioctl", &pstr)) {
+            str = pstr.buf;
+            len = pstr.len;
             if (len > IOCTL_BUFSZ) {
                 PyBuffer_Release(&pstr);
                 PyErr_SetString(PyExc_ValueError,
-                    "ioctl string arg too long");
+                                "ioctl string arg too long");
                 return NULL;
             }
-            else {
+            memcpy(buf, str, len);
-                memcpy(buf, str, len);
+            buf[len] = '\0';
-                buf[len] = '\0';
+            Py_BEGIN_ALLOW_THREADS
-                arg = buf;
+            ret = ioctl(fd, code, buf);
-            }
-        }
-        if (buf == arg) {
-            Py_BEGIN_ALLOW_THREADS /* think array.resize() */
-            ret = ioctl(fd, code, arg);
             Py_END_ALLOW_THREADS
-        }
+            if (ret < 0) {
-        else {
+                PyBuffer_Release(&pstr);
-            ret = ioctl(fd, code, arg);
+                PyErr_SetFromErrno(PyExc_IOError);
-        }
+                return NULL;
-        if (mutate_arg && (len <= IOCTL_BUFSZ)) {
+            }
-            memcpy(str, buf, len);
+            PyBuffer_Release(&pstr);
-        }
-        PyBuffer_Release(&pstr); /* No further access to str below this point */
-        if (ret < 0) {
-            PyErr_SetFromErrno(PyExc_IOError);
-            return NULL;
-        }
-        if (mutate_arg) {
-            return PyLong_FromLong(ret);
-        }
-        else {
             return PyBytes_FromStringAndSize(buf, len);
         }
-    }
 
-    PyErr_Clear();
+        PyErr_Clear();
-    if (PyArg_ParseTuple(args, "O&Is*:ioctl",
+        if (!PyArg_Parse(ob_arg,
-                         conv_descriptor, &fd, &code, &pstr)) {
+             "i;ioctl requires a file or file descriptor,"
-        str = pstr.buf;
+             " an integer and optionally an integer or buffer argument",
-        len = pstr.len;
+             &arg)) {
-        if (len > IOCTL_BUFSZ) {
+          return NULL;
-            PyBuffer_Release(&pstr);
-            PyErr_SetString(PyExc_ValueError,
-                            "ioctl string arg too long");
-            return NULL;
         }
-        memcpy(buf, str, len);
+        // Fall-through to outside the 'if' statement.
-        buf[len] = '\0';
-        Py_BEGIN_ALLOW_THREADS
-        ret = ioctl(fd, code, buf);
-        Py_END_ALLOW_THREADS
-        if (ret < 0) {
-            PyBuffer_Release(&pstr);
-            PyErr_SetFromErrno(PyExc_IOError);
-            return NULL;
-        }
-        PyBuffer_Release(&pstr);
-        return PyBytes_FromStringAndSize(buf, len);
-    }
-
-    PyErr_Clear();
-    arg = 0;
-    if (!PyArg_ParseTuple(args,
-         "O&I|i;ioctl requires a file or file descriptor,"
-         " an integer and optionally an integer or buffer argument",
-                          conv_descriptor, &fd, &code, &arg)) {
-      return NULL;
     }
     Py_BEGIN_ALLOW_THREADS
     ret = ioctl(fd, code, arg);
@@ -218,52 +269,25 @@ fcntl_ioctl(PyObject *self, PyObject *args)
 #undef IOCTL_BUFSZ
 }
 
-PyDoc_STRVAR(ioctl_doc,
+/*[clinic input]
-"ioctl(fd, op[, arg[, mutate_flag]])\n\
+fcntl.flock
-\n\
-Perform the operation op on file descriptor fd.  The values used for op\n\
-are operating system dependent, and are available as constants in the\n\
-fcntl or termios library modules, using the same names as used in the\n\
-relevant C header files.\n\
-\n\
-The argument arg is optional, and defaults to 0; it may be an int or a\n\
-buffer containing character data (most likely a string or an array). \n\
-\n\
-If the argument is a mutable buffer (such as an array) and if the\n\
-mutate_flag argument (which is only allowed in this case) is true then the\n\
-buffer is (in effect) passed to the operating system and changes made by\n\
-the OS will be reflected in the contents of the buffer after the call has\n\
-returned.  The return value is the integer returned by the ioctl system\n\
-call.\n\
-\n\
-If the argument is a mutable buffer and the mutable_flag argument is not\n\
-passed or is false, the behavior is as if a string had been passed.  This\n\
-behavior will change in future releases of Python.\n\
-\n\
-If the argument is an immutable buffer (most likely a string) then a copy\n\
-of the buffer is passed to the operating system and the return value is a\n\
-string of the same length containing whatever the operating system put in\n\
-the buffer.  The length of the arg buffer in this case is not allowed to\n\
-exceed 1024 bytes.\n\
-\n\
-If the arg given is an integer or if none is specified, the result value is\n\
-an integer corresponding to the return value of the ioctl call in the C\n\
-code.");
 
+    fd: object(type='int', converter='conv_descriptor')
+    code: int
+    /
 
-/* flock(fd, operation) */
+Perform the lock operation op on file descriptor fd.
+
+See the Unix manual page for flock(2) for details (On some systems, this
+function is emulated using fcntl()).
+[clinic start generated code]*/
 
 static PyObject *
-fcntl_flock(PyObject *self, PyObject *args)
+fcntl_flock_impl(PyModuleDef *module, int fd, int code)
+/*[clinic end generated code: output=c9035133a7dbfc96 input=b762aa9448d05e43]*/
 {
-    int fd;
-    int code;
     int ret;
 
-    if (!PyArg_ParseTuple(args, "O&i:flock",
-                          conv_descriptor, &fd, &code))
-        return NULL;
-
 #ifdef HAVE_FLOCK
     Py_BEGIN_ALLOW_THREADS
     ret = flock(fd, code);
@@ -299,29 +323,49 @@ fcntl_flock(PyObject *self, PyObject *args)
         PyErr_SetFromErrno(PyExc_IOError);
         return NULL;
     }
-    Py_INCREF(Py_None);
+    Py_RETURN_NONE;
-    return Py_None;
 }
 
-PyDoc_STRVAR(flock_doc,
-"flock(fd, operation)\n\
-\n\
-Perform the lock operation op on file descriptor fd.  See the Unix \n\
-manual page for flock(2) for details.  (On some systems, this function is\n\
-emulated using fcntl().)");
 
+/*[clinic input]
+fcntl.lockf
+
+    fd: object(type='int', converter='conv_descriptor')
+    code: int
+    lenobj: object = NULL
+    startobj: object = NULL
+    whence: int = 0
+    /
+
+A wrapper around the fcntl() locking calls.
+
+fd is the file descriptor of the file to lock or unlock, and operation is one
+of the following values:
+
+    LOCK_UN - unlock
+    LOCK_SH - acquire a shared lock
+    LOCK_EX - acquire an exclusive lock
+
+When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with
+LOCK_NB to avoid blocking on lock acquisition.  If LOCK_NB is used and the
+lock cannot be acquired, an IOError will be raised and the exception will
+have an errno attribute set to EACCES or EAGAIN (depending on the operating
+system -- for portability, check for either value).
+
+length is the number of bytes to lock, with the default meaning to lock to
+EOF.  start is the byte offset, relative to whence, to that the lock
+starts.  whence is as with fileobj.seek(), specifically:
+
+    0 - relative to the start of the file (SEEK_SET)
+    1 - relative to the current buffer position (SEEK_CUR)
+    2 - relative to the end of the file (SEEK_END)
+[clinic start generated code]*/
 
-/* lockf(fd, operation) */
 static PyObject *
-fcntl_lockf(PyObject *self, PyObject *args)
+fcntl_lockf_impl(PyModuleDef *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence)
+/*[clinic end generated code: output=5536df2892bf3ce9 input=44856fa06db36184]*/
 {
-    int fd, code, ret, whence = 0;
+    int ret;
-    PyObject *lenobj = NULL, *startobj = NULL;
-
-    if (!PyArg_ParseTuple(args, "O&i|OOi:lockf",
-                          conv_descriptor, &fd, &code,
-                          &lenobj, &startobj, &whence))
-        return NULL;
 
 #ifndef LOCK_SH
 #define LOCK_SH         1       /* shared lock */
@@ -374,43 +418,17 @@ fcntl_lockf(PyObject *self, PyObject *args)
         PyErr_SetFromErrno(PyExc_IOError);
         return NULL;
     }
-    Py_INCREF(Py_None);
+    Py_RETURN_NONE;
-    return Py_None;
 }
 
-PyDoc_STRVAR(lockf_doc,
-"lockf (fd, operation, length=0, start=0, whence=0)\n\
-\n\
-This is essentially a wrapper around the fcntl() locking calls.  fd is the\n\
-file descriptor of the file to lock or unlock, and operation is one of the\n\
-following values:\n\
-\n\
-    LOCK_UN - unlock\n\
-    LOCK_SH - acquire a shared lock\n\
-    LOCK_EX - acquire an exclusive lock\n\
-\n\
-When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n\
-LOCK_NB to avoid blocking on lock acquisition.  If LOCK_NB is used and the\n\
-lock cannot be acquired, an IOError will be raised and the exception will\n\
-have an errno attribute set to EACCES or EAGAIN (depending on the operating\n\
-system -- for portability, check for either value).\n\
-\n\
-length is the number of bytes to lock, with the default meaning to lock to\n\
-EOF.  start is the byte offset, relative to whence, to that the lock\n\
-starts.  whence is as with fileobj.seek(), specifically:\n\
-\n\
-    0 - relative to the start of the file (SEEK_SET)\n\
-    1 - relative to the current buffer position (SEEK_CUR)\n\
-    2 - relative to the end of the file (SEEK_END)");
-
 /* List of functions */
 
 static PyMethodDef fcntl_methods[] = {
-    {"fcntl",           fcntl_fcntl, METH_VARARGS, fcntl_doc},
+    FCNTL_FCNTL_METHODDEF
-    {"ioctl",           fcntl_ioctl, METH_VARARGS, ioctl_doc},
+    FCNTL_IOCTL_METHODDEF
-    {"flock",           fcntl_flock, METH_VARARGS, flock_doc},
+    FCNTL_FLOCK_METHODDEF
-    {"lockf",       fcntl_lockf, METH_VARARGS, lockf_doc},
+    FCNTL_LOCKF_METHODDEF
-    {NULL,              NULL}           /* sentinel */
+    {NULL, NULL}  /* sentinel */
 };
 
 

Parser/asdl.py

@@ -33,8 +33,7 @@ __all__ = [
 # See the EBNF at the top of the file to understand the logical connection
 # between the various node types.
 
-builtin_types = set(
+builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
-    ['identifier', 'string', 'bytes', 'int', 'object', 'singleton'])
 
 class AST:
     def __repr__(self):