From 1cf0df4f1bcc38dfd70a152af20cf584de531ea7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 25 Jan 2020 11:34:36 -0800 Subject: [PATCH] bpo-36654: Add examples for using tokenize module programmatically (GH-18187) (cherry picked from commit 4b09dc79f4d08d85f2cc945563e9c8ef1e531d7b) Co-authored-by: Windson yang --- Doc/library/tokenize.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index b208ba46d17..96778f23f8f 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -278,3 +278,22 @@ The exact token type names can be displayed using the :option:`-e` option: 4,10-4,11: RPAR ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER '' + +Example of tokenizing a file programmatically, reading unicode +strings instead of bytes with :func:`generate_tokens`:: + + import tokenize + + with tokenize.open('hello.py') as f: + tokens = tokenize.generate_tokens(f.readline) + for token in tokens: + print(token) + +Or reading bytes directly with :func:`.tokenize`:: + + import tokenize + + with open('hello.py', 'rb') as f: + tokens = tokenize.tokenize(f.readline) + for token in tokens: + print(token)