From 4b09dc79f4d08d85f2cc945563e9c8ef1e531d7b Mon Sep 17 00:00:00 2001 From: Windson yang Date: Sun, 26 Jan 2020 03:23:00 +0800 Subject: [PATCH] bpo-36654: Add examples for using tokenize module programmically (#12947) --- Doc/library/tokenize.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index b208ba46d17..96778f23f8f 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -278,3 +278,22 @@ The exact token type names can be displayed using the :option:`-e` option: 4,10-4,11: RPAR ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER '' + +Example of tokenizing a file programmatically, reading unicode +strings instead of bytes with :func:`generate_tokens`:: + + import tokenize + + with tokenize.open('hello.py') as f: + tokens = tokenize.generate_tokens(f.readline) + for token in tokens: + print(token) + +Or reading bytes directly with :func:`.tokenize`:: + + import tokenize + + with open('hello.py', 'rb') as f: + tokens = tokenize.tokenize(f.readline) + for token in tokens: + print(token)