cpython/Lib/shlex.py

336 lines
13 KiB
Python
Raw Normal View History

"""A lexical analyzer class for simple shell-like syntaxes."""
2001-01-17 04:48:39 -04:00
# Module and documentation by Eric S. Raymond, 21 Dec 1998
# Input stacking and error message cleanup added by ESR, March 2000
2001-01-17 04:48:39 -04:00
# push_source() and pop_source() made explicit by ESR, January 2001.
# Posix compliance, split(), string arguments, and
# iterator interface by Gustavo Niemeyer, April 2003.
# changes to tokenize more like Posix shells by Vinay Sajip, July 2016.
import os
import re
import sys
from collections import deque
from io import StringIO
__all__ = ["shlex", "split", "quote"]
class shlex:
2001-01-17 04:48:39 -04:00
"A lexical analyzer class for simple shell-like syntaxes."
def __init__(self, instream=None, infile=None, posix=False,
punctuation_chars=False):
if isinstance(instream, str):
instream = StringIO(instream)
2002-06-01 21:40:05 -03:00
if instream is not None:
self.instream = instream
self.infile = infile
else:
self.instream = sys.stdin
self.infile = None
self.posix = posix
if posix:
self.eof = None
else:
self.eof = ''
self.commenters = '#'
self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
if self.posix:
2010-10-27 15:52:48 -03:00
self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
self.whitespace = ' \t\r\n'
self.whitespace_split = False
self.quotes = '\'"'
self.escape = '\\'
self.escapedquotes = '"'
self.state = ' '
self.pushback = deque()
self.lineno = 1
self.debug = 0
self.token = ''
self.filestack = deque()
self.source = None
if not punctuation_chars:
punctuation_chars = ''
elif punctuation_chars is True:
punctuation_chars = '();<>|&'
self.punctuation_chars = punctuation_chars
if punctuation_chars:
# _pushback_chars is a push back queue used by lookahead logic
self._pushback_chars = deque()
# these chars added because allowed in file names, args, wildcards
self.wordchars += '~-./*?='
#remove any punctuation chars from wordchars
t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars))
self.wordchars = self.wordchars.translate(t)
def push_token(self, tok):
"Push a token onto the stack popped by the get_token method"
if self.debug >= 1:
print("shlex: pushing token " + repr(tok))
self.pushback.appendleft(tok)
def push_source(self, newstream, newfile=None):
"Push an input source onto the lexer's input source stack."
if isinstance(newstream, str):
newstream = StringIO(newstream)
self.filestack.appendleft((self.infile, self.instream, self.lineno))
self.infile = newfile
self.instream = newstream
self.lineno = 1
if self.debug:
2002-06-01 21:40:05 -03:00
if newfile is not None:
print('shlex: pushing to file %s' % (self.infile,))
else:
print('shlex: pushing to stream %s' % (self.instream,))
def pop_source(self):
"Pop the input source stack."
self.instream.close()
(self.infile, self.instream, self.lineno) = self.filestack.popleft()
if self.debug:
print('shlex: popping to %s, line %d' \
% (self.instream, self.lineno))
self.state = ' '
def get_token(self):
"Get a token from the input stream (or from stack if it's nonempty)"
if self.pushback:
tok = self.pushback.popleft()
if self.debug >= 1:
print("shlex: popping token " + repr(tok))
return tok
# No pushback. Get a token.
raw = self.read_token()
# Handle inclusions
if self.source is not None:
while raw == self.source:
spec = self.sourcehook(self.read_token())
if spec:
(newfile, newstream) = spec
self.push_source(newstream, newfile)
raw = self.get_token()
# Maybe we got EOF instead?
while raw == self.eof:
if not self.filestack:
return self.eof
else:
self.pop_source()
raw = self.get_token()
# Neither inclusion nor EOF
if self.debug >= 1:
if raw != self.eof:
print("shlex: token=" + repr(raw))
else:
print("shlex: token=EOF")
return raw
def read_token(self):
quoted = False
escapedstate = ' '
while True:
if self.punctuation_chars and self._pushback_chars:
nextchar = self._pushback_chars.pop()
else:
nextchar = self.instream.read(1)
if nextchar == '\n':
self.lineno += 1
if self.debug >= 3:
print("shlex: in state %r I see character: %r" % (self.state,
nextchar))
if self.state is None:
self.token = '' # past end of file
break
elif self.state == ' ':
if not nextchar:
self.state = None # end of file
break
elif nextchar in self.whitespace:
if self.debug >= 2:
print("shlex: I see whitespace in whitespace state")
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif nextchar in self.commenters:
self.instream.readline()
self.lineno += 1
elif self.posix and nextchar in self.escape:
escapedstate = 'a'
self.state = nextchar
elif nextchar in self.wordchars:
self.token = nextchar
self.state = 'a'
elif nextchar in self.punctuation_chars:
self.token = nextchar
self.state = 'c'
elif nextchar in self.quotes:
if not self.posix:
self.token = nextchar
self.state = nextchar
elif self.whitespace_split:
self.token = nextchar
self.state = 'a'
else:
self.token = nextchar
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif self.state in self.quotes:
quoted = True
if not nextchar: # end of file
if self.debug >= 2:
print("shlex: I see EOF in quotes state")
# XXX what error should be raised here?
2007-08-29 22:19:48 -03:00
raise ValueError("No closing quotation")
if nextchar == self.state:
if not self.posix:
self.token += nextchar
self.state = ' '
break
else:
self.state = 'a'
elif (self.posix and nextchar in self.escape and self.state
in self.escapedquotes):
escapedstate = self.state
self.state = nextchar
else:
self.token += nextchar
elif self.state in self.escape:
if not nextchar: # end of file
if self.debug >= 2:
print("shlex: I see EOF in escape state")
# XXX what error should be raised here?
2007-08-29 22:19:48 -03:00
raise ValueError("No escaped character")
# In posix shells, only the quote itself or the escape
# character may be escaped within quotes.
if (escapedstate in self.quotes and
nextchar != self.state and nextchar != escapedstate):
self.token += self.state
self.token += nextchar
self.state = escapedstate
elif self.state in ('a', 'c'):
if not nextchar:
2001-01-17 04:48:39 -04:00
self.state = None # end of file
break
elif nextchar in self.whitespace:
if self.debug >= 2:
print("shlex: I see whitespace in word state")
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif nextchar in self.commenters:
self.instream.readline()
self.lineno += 1
if self.posix:
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
elif self.posix and nextchar in self.quotes:
self.state = nextchar
elif self.posix and nextchar in self.escape:
escapedstate = 'a'
self.state = nextchar
elif self.state == 'c':
if nextchar in self.punctuation_chars:
self.token += nextchar
else:
if nextchar not in self.whitespace:
self._pushback_chars.append(nextchar)
self.state = ' '
break
elif (nextchar in self.wordchars or nextchar in self.quotes
or self.whitespace_split):
self.token += nextchar
else:
if self.punctuation_chars:
self._pushback_chars.append(nextchar)
else:
self.pushback.appendleft(nextchar)
if self.debug >= 2:
print("shlex: I see punctuation in word state")
self.state = ' '
if self.token or (self.posix and quoted):
break # emit current token
else:
continue
result = self.token
self.token = ''
if self.posix and not quoted and result == '':
result = None
if self.debug > 1:
if result:
print("shlex: raw token=" + repr(result))
else:
print("shlex: raw token=EOF")
return result
def sourcehook(self, newfile):
"Hook called on a filename to be sourced."
if newfile[0] == '"':
newfile = newfile[1:-1]
# This implements cpp-like semantics for relative-path inclusion.
if isinstance(self.infile, str) and not os.path.isabs(newfile):
newfile = os.path.join(os.path.dirname(self.infile), newfile)
return (newfile, open(newfile, "r"))
def error_leader(self, infile=None, lineno=None):
"Emit a C-compiler-like, Emacs-friendly error-message leader."
2002-06-01 21:40:05 -03:00
if infile is None:
infile = self.infile
2002-06-01 21:40:05 -03:00
if lineno is None:
lineno = self.lineno
return "\"%s\", line %d: " % (infile, lineno)
def __iter__(self):
return self
def __next__(self):
token = self.get_token()
if token == self.eof:
raise StopIteration
return token
Merged revisions 55631-55794 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/p3yk ................ r55636 | neal.norwitz | 2007-05-29 00:06:39 -0700 (Tue, 29 May 2007) | 149 lines Merged revisions 55506-55635 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r55507 | georg.brandl | 2007-05-22 07:28:17 -0700 (Tue, 22 May 2007) | 2 lines Remove the "panel" module doc file which has been ignored since 1994. ........ r55522 | mark.hammond | 2007-05-22 19:04:28 -0700 (Tue, 22 May 2007) | 4 lines Remove definition of PY_UNICODE_TYPE from pyconfig.h, allowing the definition in unicodeobject.h to be used, giving us the desired wchar_t in place of 'unsigned short'. As discussed on python-dev. ........ r55525 | neal.norwitz | 2007-05-22 23:35:32 -0700 (Tue, 22 May 2007) | 6 lines Add -3 option to the interpreter to warn about features that are deprecated and will be changed/removed in Python 3.0. This patch is mostly from Anthony. I tweaked some format and added a little doc. ........ r55527 | neal.norwitz | 2007-05-22 23:57:35 -0700 (Tue, 22 May 2007) | 1 line Whitespace cleanup ........ r55528 | neal.norwitz | 2007-05-22 23:58:36 -0700 (Tue, 22 May 2007) | 1 line Add a bunch more deprecation warnings for builtins that are going away in 3.0 ........ r55549 | georg.brandl | 2007-05-24 09:49:29 -0700 (Thu, 24 May 2007) | 2 lines shlex.split() now has an optional "posix" parameter. ........ r55550 | georg.brandl | 2007-05-24 10:33:33 -0700 (Thu, 24 May 2007) | 2 lines Fix parameter passing. ........ r55555 | facundo.batista | 2007-05-24 10:50:54 -0700 (Thu, 24 May 2007) | 6 lines Added an optional timeout parameter to urllib.ftpwrapper, with tests (for this and a basic one, because there weren't any). Changed also NEWS, but didn't find documentation for this function, assumed it wasn't public... ........ r55563 | facundo.batista | 2007-05-24 13:01:59 -0700 (Thu, 24 May 2007) | 4 lines Removed the .recv() in the test, is not necessary, and was causing problems that didn't have anything to do with was actually being tested... ........ r55564 | facundo.batista | 2007-05-24 13:51:19 -0700 (Thu, 24 May 2007) | 5 lines Let's see if reading exactly what is written allow this live test to pass (now I know why there were so few tests in ftp, http, etc, :( ). ........ r55567 | facundo.batista | 2007-05-24 20:10:28 -0700 (Thu, 24 May 2007) | 4 lines Trying to make the tests work in Windows and Solaris, everywhere else just works ........ r55568 | facundo.batista | 2007-05-24 20:47:19 -0700 (Thu, 24 May 2007) | 4 lines Fixing stupid error, and introducing a sleep, to see if the other thread is awakened and finish sending data. ........ r55569 | facundo.batista | 2007-05-24 21:20:22 -0700 (Thu, 24 May 2007) | 4 lines Commenting out the tests until find out who can test them in one of the problematic enviroments. ........ r55570 | neal.norwitz | 2007-05-24 22:13:40 -0700 (Thu, 24 May 2007) | 2 lines Get test passing again by commenting out the reference to the test class. ........ r55575 | vinay.sajip | 2007-05-25 00:05:59 -0700 (Fri, 25 May 2007) | 1 line Updated docstring for SysLogHandler (#1720726). ........ r55576 | vinay.sajip | 2007-05-25 00:06:55 -0700 (Fri, 25 May 2007) | 1 line Updated documentation for SysLogHandler (#1720726). ........ r55592 | brett.cannon | 2007-05-25 13:17:15 -0700 (Fri, 25 May 2007) | 3 lines Remove direct call's to file's constructor and replace them with calls to open() as ths is considered best practice. ........ r55601 | kristjan.jonsson | 2007-05-26 12:19:50 -0700 (Sat, 26 May 2007) | 1 line Remove the rgbimgmodule from PCBuild8 ........ r55602 | kristjan.jonsson | 2007-05-26 12:31:39 -0700 (Sat, 26 May 2007) | 1 line Include <windows.h> after python.h, so that WINNT is properly set before windows.h is included. Fixes warnings in PC builds. ........ r55603 | walter.doerwald | 2007-05-26 14:04:13 -0700 (Sat, 26 May 2007) | 2 lines Fix typo. ........ r55604 | peter.astrand | 2007-05-26 15:18:20 -0700 (Sat, 26 May 2007) | 1 line Applied patch 1669481, slightly modified: Support close_fds on Win32 ........ r55606 | neal.norwitz | 2007-05-26 21:08:54 -0700 (Sat, 26 May 2007) | 2 lines Add the new function object attribute names from py3k. ........ r55617 | lars.gustaebel | 2007-05-27 12:49:30 -0700 (Sun, 27 May 2007) | 20 lines Added errors argument to TarFile class that allows the user to specify an error handling scheme for character conversion. Additional scheme "utf-8" in read mode. Unicode input filenames are now supported by design. The values of the pax_headers dictionary are now limited to unicode objects. Fixed: The prefix field is no longer used in PAX_FORMAT (in conformance with POSIX). Fixed: In read mode use a possible pax header size field. Fixed: Strip trailing slashes from pax header name values. Fixed: Give values in user-specified pax_headers precedence when writing. Added unicode tests. Added pax/regtype4 member to testtar.tar all possible number fields in a pax header. Added two chapters to the documentation about the different formats tarfile.py supports and how unicode issues are handled. ........ r55618 | raymond.hettinger | 2007-05-27 22:23:22 -0700 (Sun, 27 May 2007) | 1 line Explain when groupby() issues a new group. ........ r55634 | martin.v.loewis | 2007-05-28 21:01:29 -0700 (Mon, 28 May 2007) | 2 lines Test pre-commit hook for a link to a .py file. ........ r55635 | martin.v.loewis | 2007-05-28 21:02:03 -0700 (Mon, 28 May 2007) | 2 lines Revert 55634. ........ ................ r55639 | neal.norwitz | 2007-05-29 00:58:11 -0700 (Tue, 29 May 2007) | 1 line Remove sys.exc_{type,exc_value,exc_traceback} ................ r55641 | neal.norwitz | 2007-05-29 01:03:50 -0700 (Tue, 29 May 2007) | 1 line Missed one sys.exc_type. I wonder why exc_{value,traceback} were already gone ................ r55642 | neal.norwitz | 2007-05-29 01:08:33 -0700 (Tue, 29 May 2007) | 1 line Missed more doc for sys.exc_* attrs. ................ r55643 | neal.norwitz | 2007-05-29 01:18:19 -0700 (Tue, 29 May 2007) | 1 line Remove sys.exc_clear() ................ r55665 | guido.van.rossum | 2007-05-29 19:45:43 -0700 (Tue, 29 May 2007) | 4 lines Make None, True, False keywords. We can now also delete all the other places that explicitly forbid assignment to None, but I'm not going to bother right now. ................ r55666 | guido.van.rossum | 2007-05-29 20:01:51 -0700 (Tue, 29 May 2007) | 3 lines Found another place that needs check for forbidden names. Fixed test_syntax.py accordingly (it helped me find that one). ................ r55668 | guido.van.rossum | 2007-05-29 20:41:48 -0700 (Tue, 29 May 2007) | 2 lines Mark None, True, False as keywords. ................ r55673 | neal.norwitz | 2007-05-29 23:28:25 -0700 (Tue, 29 May 2007) | 3 lines Get the dis module working on modules again after changing dicts to not return lists and also new-style classes. Add a test. ................ r55674 | neal.norwitz | 2007-05-29 23:35:45 -0700 (Tue, 29 May 2007) | 1 line Umm, it helps to add the module that the test uses ................ r55675 | neal.norwitz | 2007-05-29 23:53:05 -0700 (Tue, 29 May 2007) | 4 lines Try to fix up all the other places that were assigning to True/False. There's at least one more problem in test.test_xmlrpc. I have other changes in that file and that should be fixed soon (I hope). ................ r55679 | neal.norwitz | 2007-05-30 00:31:55 -0700 (Wed, 30 May 2007) | 1 line Fix up another place that was assigning to True/False. ................ r55688 | brett.cannon | 2007-05-30 14:19:47 -0700 (Wed, 30 May 2007) | 2 lines Ditch MimeWriter. ................ r55692 | brett.cannon | 2007-05-30 14:52:00 -0700 (Wed, 30 May 2007) | 2 lines Remove the mimify module. ................ r55707 | guido.van.rossum | 2007-05-31 05:08:45 -0700 (Thu, 31 May 2007) | 2 lines Backport the addition of show_code() to dis.py -- it's too handy. ................ r55708 | guido.van.rossum | 2007-05-31 06:22:57 -0700 (Thu, 31 May 2007) | 7 lines Fix a fairly long-standing bug in the check for assignment to None (and other keywords, these days). In 2.5, you could write foo(None=1) without getting a SyntaxError (although foo()'s definition would have to use **kwds to avoid getting a runtime error complaining about an unknown keyword of course). This ought to be backported to 2.5.2 or at least 2.6. ................ r55724 | brett.cannon | 2007-05-31 19:32:41 -0700 (Thu, 31 May 2007) | 2 lines Remove the cfmfile. ................ r55727 | neal.norwitz | 2007-05-31 22:19:44 -0700 (Thu, 31 May 2007) | 1 line Remove reload() builtin. ................ r55729 | neal.norwitz | 2007-05-31 22:51:30 -0700 (Thu, 31 May 2007) | 59 lines Merged revisions 55636-55728 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r55637 | georg.brandl | 2007-05-29 00:16:47 -0700 (Tue, 29 May 2007) | 2 lines Fix rst markup. ........ r55638 | neal.norwitz | 2007-05-29 00:51:39 -0700 (Tue, 29 May 2007) | 1 line Fix typo in doc ........ r55671 | neal.norwitz | 2007-05-29 21:53:41 -0700 (Tue, 29 May 2007) | 1 line Fix indentation (whitespace only). ........ r55676 | thomas.heller | 2007-05-29 23:58:30 -0700 (Tue, 29 May 2007) | 1 line Fix compiler warnings. ........ r55677 | thomas.heller | 2007-05-30 00:01:25 -0700 (Wed, 30 May 2007) | 2 lines Correct the name of a field in the WIN32_FIND_DATAA and WIN32_FIND_DATAW structures. Closes bug #1726026. ........ r55686 | brett.cannon | 2007-05-30 13:46:26 -0700 (Wed, 30 May 2007) | 2 lines Have MimeWriter raise a DeprecationWarning as per PEP 4 and its documentation. ........ r55690 | brett.cannon | 2007-05-30 14:48:58 -0700 (Wed, 30 May 2007) | 3 lines Have mimify raise a DeprecationWarning. The docs and PEP 4 have listed the module as deprecated for a while. ........ r55696 | brett.cannon | 2007-05-30 15:24:28 -0700 (Wed, 30 May 2007) | 2 lines Have md5 raise a DeprecationWarning as per PEP 4. ........ r55705 | neal.norwitz | 2007-05-30 21:14:22 -0700 (Wed, 30 May 2007) | 1 line Add some spaces in the example code. ........ r55716 | brett.cannon | 2007-05-31 12:20:00 -0700 (Thu, 31 May 2007) | 2 lines Have the sha module raise a DeprecationWarning as specified in PEP 4. ........ r55719 | brett.cannon | 2007-05-31 12:40:42 -0700 (Thu, 31 May 2007) | 2 lines Cause buildtools to raise a DeprecationWarning. ........ r55721 | brett.cannon | 2007-05-31 13:01:11 -0700 (Thu, 31 May 2007) | 2 lines Have cfmfile raise a DeprecationWarning as per PEP 4. ........ r55726 | neal.norwitz | 2007-05-31 21:56:47 -0700 (Thu, 31 May 2007) | 1 line Mail if there is an installation failure. ........ ................ r55730 | neal.norwitz | 2007-05-31 23:22:07 -0700 (Thu, 31 May 2007) | 2 lines Remove the code that was missed in rev 55303. ................ r55738 | neal.norwitz | 2007-06-01 19:10:43 -0700 (Fri, 01 Jun 2007) | 1 line Fix doc breakage ................ r55741 | neal.norwitz | 2007-06-02 00:41:58 -0700 (Sat, 02 Jun 2007) | 1 line Remove timing module (plus some remnants of other modules). ................ r55742 | neal.norwitz | 2007-06-02 00:51:44 -0700 (Sat, 02 Jun 2007) | 1 line Remove posixfile module (plus some remnants of other modules). ................ r55744 | neal.norwitz | 2007-06-02 10:18:56 -0700 (Sat, 02 Jun 2007) | 1 line Fix doc breakage. ................ r55745 | neal.norwitz | 2007-06-02 11:32:16 -0700 (Sat, 02 Jun 2007) | 1 line Make a whatsnew 3.0 template. ................ r55754 | neal.norwitz | 2007-06-03 23:24:18 -0700 (Sun, 03 Jun 2007) | 1 line SF #1730441, os._execvpe raises UnboundLocal due to new try/except semantics ................ r55755 | neal.norwitz | 2007-06-03 23:26:00 -0700 (Sun, 03 Jun 2007) | 1 line Get rid of extra whitespace ................ r55794 | guido.van.rossum | 2007-06-06 15:29:22 -0700 (Wed, 06 Jun 2007) | 3 lines Make this compile in GCC 2.96, which does not allow interspersing declarations and code. ................
2007-06-06 20:52:48 -03:00
def split(s, comments=False, posix=True):
lex = shlex(s, posix=posix)
lex.whitespace_split = True
if not comments:
lex.commenters = ''
return list(lex)
2011-08-16 13:03:41 -03:00
_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search
def quote(s):
"""Return a shell-escaped version of the string *s*."""
if not s:
return "''"
if _find_unsafe(s) is None:
return s
# use single quotes, and put single quotes into double quotes
# the string $'b is then quoted as '$'"'"'b'
return "'" + s.replace("'", "'\"'\"'") + "'"
def _print_tokens(lexer):
while 1:
tt = lexer.get_token()
if not tt:
break
print("Token: " + repr(tt))
if __name__ == '__main__':
if len(sys.argv) == 1:
_print_tokens(shlex())
else:
fn = sys.argv[1]
with open(fn) as f:
_print_tokens(shlex(f, fn))