Lib/shlex.py

   1 # Module and documentation by Eric S. Raymond, 21 Dec 1998
   2
   3 import sys
   4
   5 class shlex:
   6     "A lexical analyzer class for simple shell-like syntaxes."
   7     def __init__(self, instream=None):
   8         if instream:
   9             self.instream = instream
  10         else:
  11             self.instream = sys.stdin
  12         self.commenters = '#'
  13         self.wordchars = 'abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
  14         self.whitespace = ' \t\r\n'
  15         self.quotes = '\'"'
  16         self.state = ' '
  17         self.pushback = [];
  18         self.lineno = 1
  19         self.debug = 0
  20         self.token = ''
  21
  22     def push_token(self, tok):
  23         "Push a token onto the stack popped by the get_token method"
  24         if (self.debug >= 1):
  25             print "Pushing " + tok
  26         self.pushback = [tok] + self.pushback;
  27
  28     def get_token(self):
  29         "Get a token from the input stream (or from stack if it's monempty)"
  30         if self.pushback:
  31             tok = self.pushback[0]
  32             self.pushback = self.pushback[1:]
  33             if (self.debug >= 1):
  34                 print "Popping " + tok
  35             return tok
  36         tok = ''
  37         while 1:
  38             nextchar = self.instream.read(1);
  39             if nextchar == '\n':
  40                 self.lineno = self.lineno + 1
  41             if self.debug >= 3:
  42                 print "In state " + repr(self.state) + " I see character: " + repr(nextchar)
  43             if self.state == None:
  44                 return ''
  45             elif self.state == ' ':
  46                 if not nextchar:
  47                     self.state = None;  # end of file
  48                     break
  49                 elif nextchar in self.whitespace:
  50                     if self.debug >= 2:
  51                         print "I see whitespace in whitespace state"
  52                     if self.token:
  53                         break   # emit current token
  54                     else:
  55                         continue
  56                 elif nextchar in self.commenters:
  57                     self.instream.readline()
  58                     self.lineno = self.lineno + 1
  59                 elif nextchar in self.wordchars:
  60                     self.token = nextchar
  61                     self.state = 'a'
  62                 elif nextchar in self.quotes:
  63                     self.token = nextchar
  64                     self.state = nextchar
  65                 else:
  66                     self.token = nextchar
  67                     if self.token:
  68                         break   # emit current token
  69                     else:
  70                         continue
  71             elif self.state in self.quotes:
  72                 self.token = self.token + nextchar
  73                 if nextchar == self.state:
  74                     self.state = ' '
  75                     break
  76             elif self.state == 'a':
  77                 if not nextchar:
  78                     self.state = None;  # end of file
  79                     break
  80                 elif nextchar in self.whitespace:
  81                     if self.debug >= 2:
  82                         print "I see whitespace in word state"
  83                     self.state = ' '
  84                     if self.token:
  85                         break   # emit current token
  86                     else:
  87                         continue
  88                 elif nextchar in self.commenters:
  89                     self.instream.readline()
  90                     self.lineno = self.lineno + 1
  91                 elif nextchar in self.wordchars or nextchar in self.quotes:
  92                     self.token = self.token + nextchar
  93                 else:
  94                     self.pushback = [nextchar] + self.pushback
  95                     if self.debug >= 2:
  96                         print "I see punctuation in word state"
  97                     self.state = ' '
  98                     if self.token:
  99                         break   # emit current token
 100                     else:
 101                         continue
 102
 103         result = self.token
 104         self.token = ''
 105         if self.debug >= 1:
 106             print "Token: " + result
 107         return result
 108
 109 if __name__ == '__main__':
 110
 111     lexer = shlex()
 112     while 1:
 113         tt = lexer.get_token()
 114         if tt != None:
 115             print "Token: " + repr(tt)
 116         else:
 117             break
 118