diff --git a/Lib/shlex.py b/Lib/shlex.py index de9b9740b1f..2d7d68a79a7 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -1,14 +1,14 @@ """A lexical analyzer class for simple shell-like syntaxes.""" -# Module and documentation by Eric S. Raymond, 21 Dec 1998 +# Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 +# push_source() and pop_source() made explicit by ESR, January 2001. import os.path import sys - class shlex: - "A lexical analyzer class for simple shell-like syntaxes." + "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None): if instream: self.instream = instream @@ -38,6 +38,28 @@ class shlex: print "shlex: pushing token " + `tok` self.pushback = [tok] + self.pushback + def push_source(self, newstream, newfile=None): + "Push an input source onto the lexer's input source stack." + self.filestack.insert(0, (self.infile, self.instream, self.lineno)) + self.infile = newfile + self.instream = newstream + self.lineno = 1 + if self.debug: + if newfile: + print 'shlex: pushing to file %s' % (self.infile,) + else: + print 'shlex: pushing to stream %s' % (self.instream,) + + def pop_source(self): + "Pop the input source stack." + self.instream.close() + (self.infile, self.instream, self.lineno) = self.filestack[0] + self.filestack = self.filestack[1:] + if self.debug: + print 'shlex: popping to %s, line %d' \ + % (self.instream, self.lineno) + self.state = ' ' + def get_token(self): "Get a token from the input stream (or from stack if it's nonempty)" if self.pushback: @@ -50,26 +72,17 @@ class shlex: raw = self.read_token() # Handle inclusions while raw == self.source: - (newfile, newstream) = self.sourcehook(self.read_token()) - self.filestack.insert(0, (self.infile, self.instream, self.lineno)) - self.infile = newfile - self.instream = newstream - self.lineno = 1 - if self.debug: - print 'shlex: pushing to file %s' % (self.infile,) + spec = self.sourcehook(self.read_token()) + if spec: + (newfile, newstream) = spec + self.push_source(newstream, newfile) raw = self.get_token() # Maybe we got EOF instead? while raw == "": if len(self.filestack) == 0: return "" else: - self.instream.close() - (self.infile, self.instream, self.lineno) = self.filestack[0] - self.filestack = self.filestack[1:] - if self.debug: - print 'shlex: popping to %s, line %d' \ - % (self.instream, self.lineno) - self.state = ' ' + self.pop_source() raw = self.get_token() # Neither inclusion nor EOF if self.debug >= 1: @@ -88,13 +101,13 @@ class shlex: self.lineno = self.lineno + 1 if self.debug >= 3: print "shlex: in state", repr(self.state), \ - "I see character:", repr(nextchar) + "I see character:", repr(nextchar) if self.state is None: - self.token = '' # past end of file + self.token = '' # past end of file break elif self.state == ' ': if not nextchar: - self.state = None # end of file + self.state = None # end of file break elif nextchar in self.whitespace: if self.debug >= 2: @@ -123,14 +136,14 @@ class shlex: if nextchar == self.state: self.state = ' ' break - elif not nextchar: # end of file + elif not nextchar: # end of file if self.debug >= 2: print "shlex: I see EOF in quotes state" # XXX what error should be raised here? raise ValueError, "No closing quotation" elif self.state == 'a': if not nextchar: - self.state = None # end of file + self.state = None # end of file break elif nextchar in self.whitespace: if self.debug >= 2: @@ -181,7 +194,7 @@ class shlex: return "\"%s\", line %d: " % (infile, lineno) -if __name__ == '__main__': +if __name__ == '__main__': if len(sys.argv) == 1: lexer = shlex() else: