bpo-33475: Fix and improve converting annotations to strings. (GH-6774)

Include/ast.h

@@ -19,9 +19,7 @@ PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
 #ifndef Py_LIMITED_API
 
 /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
-PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(
+PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty);
-    expr_ty e,
-    int omit_parens);
 
 #endif /* !Py_LIMITED_API */
 

Lib/test/test_future.py

@@ -157,55 +157,76 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq('True or False or None')
         eq('True and False')
         eq('True and False and None')
-        eq('(Name1 and Name2) or Name3')
+        eq('Name1 and Name2 or Name3')
-        eq('Name1 or (Name2 and Name3)')
+        eq('Name1 and (Name2 or Name3)')
-        eq('(Name1 and Name2) or (Name3 and Name4)')
+        eq('Name1 or Name2 and Name3')
-        eq('Name1 or (Name2 and Name3) or Name4')
+        eq('(Name1 or Name2) and Name3')
+        eq('Name1 and Name2 or Name3 and Name4')
+        eq('Name1 or Name2 and Name3 or Name4')
+        eq('a + b + (c + d)')
+        eq('a * b * (c * d)')
+        eq('(a ** b) ** c ** d')
         eq('v1 << 2')
         eq('1 >> v2')
-        eq(r'1 % finished')
+        eq('1 % finished')
-        eq('((1 + v2) - (v3 * 4)) ^ (((5 ** v6) / 7) // 8)')
+        eq('1 + v2 - v3 * 4 ^ 5 ** v6 / 7 // 8')
         eq('not great')
+        eq('not not great')
         eq('~great')
         eq('+value')
+        eq('++value')
         eq('-1')
-        eq('(~int) and (not ((v1 ^ (123 + v2)) | True))')
+        eq('~int and not v1 ^ 123 + v2 | True')
+        eq('a + (not b)')
         eq('lambda arg: None')
         eq('lambda a=True: a')
         eq('lambda a, b, c=True: a')
-        eq("lambda a, b, c=True, *, d=(1 << v2), e='str': a")
+        eq("lambda a, b, c=True, *, d=1 << v2, e='str': a")
-        eq("lambda a, b, c=True, *vararg, d=(v1 << 2), e='str', **kwargs: a + b")
+        eq("lambda a, b, c=True, *vararg, d=v1 << 2, e='str', **kwargs: a + b")
+        eq('lambda x: lambda y: x + y')
         eq('1 if True else 2')
-        eq('(str or None) if True else (str or bytes or None)')
+        eq('str or None if int or True else str or bytes or None')
-        eq('(str or None) if (1 if True else 2) else (str or bytes or None)')
+        eq('str or None if (1 if True else 2) else str or bytes or None')
-        eq("{'2.7': dead, '3.7': (long_live or die_hard)}")
+        eq("0 if not x else 1 if x > 0 else -1")
-        eq("{'2.7': dead, '3.7': (long_live or die_hard), **{'3.6': verygood}}")
+        eq("(1 if x > 0 else -1) if x else 0")
+        eq("{'2.7': dead, '3.7': long_live or die_hard}")
+        eq("{'2.7': dead, '3.7': long_live or die_hard, **{'3.6': verygood}}")
         eq("{**a, **b, **c}")
-        eq("{'2.7', '3.6', '3.7', '3.8', '3.9', ('4.0' if gilectomy else '3.10')}")
+        eq("{'2.7', '3.6', '3.7', '3.8', '3.9', '4.0' if gilectomy else '3.10'}")
-        eq("({'a': 'b'}, (True or False), (+value), 'string', b'bytes') or None")
+        eq("{*a, *b, *c}")
+        eq("({'a': 'b'}, True or False, +value, 'string', b'bytes') or None")
         eq("()")
-        eq("(1,)")
+        eq("(a,)")
-        eq("(1, 2)")
+        eq("(a, b)")
-        eq("(1, 2, 3)")
+        eq("(a, b, c)")
+        eq("(*a, *b, *c)")
         eq("[]")
-        eq("[1, 2, 3, 4, 5, 6, 7, 8, 9, (10 or A), (11 or B), (12 or C)]")
+        eq("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C]")
+        eq("[*a, *b, *c]")
         eq("{i for i in (1, 2, 3)}")
-        eq("{(i ** 2) for i in (1, 2, 3)}")
+        eq("{i ** 2 for i in (1, 2, 3)}")
-        eq("{(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))}")
+        eq("{i ** 2 for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))}")
-        eq("{((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)}")
+        eq("{i ** 2 + j for i in (1, 2, 3) for j in (1, 2, 3)}")
         eq("[i for i in (1, 2, 3)]")
-        eq("[(i ** 2) for i in (1, 2, 3)]")
+        eq("[i ** 2 for i in (1, 2, 3)]")
-        eq("[(i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))]")
+        eq("[i ** 2 for i, _ in ((1, 'a'), (2, 'b'), (3, 'c'))]")
-        eq("[((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3)]")
+        eq("[i ** 2 + j for i in (1, 2, 3) for j in (1, 2, 3)]")
-        eq(r"{i: 0 for i in (1, 2, 3)}")
+        eq("(i for i in (1, 2, 3))")
+        eq("(i ** 2 for i in (1, 2, 3))")
+        eq("(i ** 2 for i, _ in ((1, 'a'), (2, 'b'), (3, 'c')))")
+        eq("(i ** 2 + j for i in (1, 2, 3) for j in (1, 2, 3))")
+        eq("{i: 0 for i in (1, 2, 3)}")
         eq("{i: j for i, j in ((1, 'a'), (2, 'b'), (3, 'c'))}")
+        eq("[(x, y) for x, y in (a, b)]")
+        eq("[(x,) for x, in (a,)]")
         eq("Python3 > Python2 > COBOL")
         eq("Life is Life")
         eq("call()")
         eq("call(arg)")
         eq("call(kwarg='hey')")
         eq("call(arg, kwarg='hey')")
-        eq("call(arg, another, kwarg='hey', **kwargs)")
+        eq("call(arg, *args, another, kwarg='hey')")
+        eq("call(arg, another, kwarg='hey', **kwargs, kwarg2='ho')")
         eq("lukasz.langa.pl")
         eq("call.me(maybe)")
         eq("1 .real")
@@ -213,6 +234,7 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq("....__class__")
         eq("list[str]")
         eq("dict[str, int]")
+        eq("set[str,]")
         eq("tuple[str, ...]")
         eq("tuple[str, int, float, dict[str, int]]")
         eq("slice[0]")
@@ -222,49 +244,28 @@ class AnnotationsFutureTestCase(unittest.TestCase):
         eq("slice[:-1]")
         eq("slice[1:]")
         eq("slice[::-1]")
-        eq('(str or None) if (sys.version_info[0] > (3,)) else (str or bytes or None)')
+        eq("slice[()]")
+        eq("slice[a, b:c, d:e:f]")
+        eq("slice[(x for x in a)]")
+        eq('str or None if sys.version_info[0] > (3,) else str or bytes or None')
         eq("f'f-string without formatted values is just a string'")
         eq("f'{{NOT a formatted value}}'")
         eq("f'some f-string with {a} {few():.2f} {formatted.values!r}'")
         eq('''f"{f'{nested} inner'} outer"''')
         eq("f'space between opening braces: { {a for a in (1, 2, 3)}}'")
-
+        eq("f'{(lambda x: x)}'")
-    def test_annotations_inexact(self):
+        eq("f'{(None if a else lambda x: x)}'")
-        """Source formatting is not always preserved
-
-        This is due to reconstruction from AST.  We *need to* put the parens
-        in nested expressions because we don't know if the source code
-        had them in the first place or not.
-        """
-        eq = partial(self.assertAnnotationEqual, drop_parens=True)
-        eq('Name1 and Name2 or Name3')
-        eq('Name1 or Name2 and Name3')
-        eq('Name1 and Name2 or Name3 and Name4')
-        eq('Name1 or Name2 and Name3 or Name4')
-        eq('1 + v2 - v3 * 4 ^ v5 ** 6 / 7 // 8')
-        eq('~int and not v1 ^ 123 + v2 | True')
-        eq('str or None if True else str or bytes or None')
-        eq("{'2.7': dead, '3.7': long_live or die_hard}")
-        eq("{'2.7', '3.6', '3.7', '3.8', '3.9', '4.0' if gilectomy else '3.10'}")
-        eq("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10 or A, 11 or B, 12 or C]")
-        # Consequently, we always drop unnecessary parens if they were given in
-        # the outer scope:
-        some_name = self.getActual("(SomeName)")
-        self.assertEqual(some_name, 'SomeName')
-        # Interestingly, in the case of tuples (and generator expressions) the
-        # parens are *required* by the Python syntax in the annotation context.
-        # But there's no point storing that detail in __annotations__ so we're
-        # fine with the parens-less form.
-        eq = partial(self.assertAnnotationEqual, is_tuple=True)
-        eq("(Good, Bad, Ugly)")
-        eq("(i for i in (1, 2, 3))")
-        eq("((i ** 2) for i in (1, 2, 3))")
-        eq("((i ** 2) for i, _ in ((1, 'a'), (2, 'b'), (3, 'c')))")
-        eq("(((i ** 2) + j) for i in (1, 2, 3) for j in (1, 2, 3))")
-        eq("(*starred)")
         eq('(yield from outside_of_generator)')
         eq('(yield)')
-        eq('(await some.complicated[0].call(with_args=(True or (1 is not 1))))')
+        eq('(yield a + b)')
+        eq('await some.complicated[0].call(with_args=True or 1 is not 1)')
+        eq('[x for x in (a if b else c)]')
+        eq('[x for x in a if (b if c else d)]')
+        eq('f(x for x in a)')
+        eq('f(1, (x for x in a))')
+        eq('f((x for x in a), 2)')
+        eq('(((a)))', 'a')
+        eq('(((a, b)))', '(a, b)')
 
 
 if __name__ == "__main__":

Misc/NEWS.d/next/Core and Builtins/2018-05-13-01-26-18.bpo-33475.rI0y1U.rst

@@ -0,0 +1,2 @@
+Fixed miscellaneous bugs in converting annotations to strings and optimized
+parentheses in the string representation.

Python/compile.c

@@ -1822,7 +1822,7 @@ error:
 static int
 compiler_visit_annexpr(struct compiler *c, expr_ty annotation)
 {
-    ADDOP_LOAD_CONST_NEW(c, _PyAST_ExprAsUnicode(annotation, 1));
+    ADDOP_LOAD_CONST_NEW(c, _PyAST_ExprAsUnicode(annotation));
     return 1;
 }