home *** CD-ROM | disk | FTP | other *** search
/ Chip 2003 January / Chip_2003-01_cd2.bin / convert / eJayMp3Pro / mp3pro_demo.exe / SHLEX.PY < prev    next >
Encoding:
Python Source  |  1999-04-13  |  4.2 KB  |  119 lines
  1. # Module and documentation by Eric S. Raymond, 21 Dec 1998 
  2.  
  3. import sys, os, string
  4.  
  5. class shlex:
  6.     "A lexical analyzer class for simple shell-like syntaxes." 
  7.     def __init__(self, instream=None):
  8.         if instream:
  9.             self.instream = instream
  10.         else:
  11.             self.instream = sys.stdin
  12.         self.commenters = '#'
  13.         self.wordchars = 'abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
  14.         self.whitespace = ' \t\r\n'
  15.         self.quotes = '\'"'
  16.         self.state = ' '
  17.         self.pushback = [];
  18.         self.lineno = 1
  19.         self.debug = 0
  20.         self.token = ''
  21.  
  22.     def push_token(self, tok):
  23.         "Push a token onto the stack popped by the get_token method"
  24.         if (self.debug >= 1):
  25.             print "Pushing " + tok
  26.         self.pushback = [tok] + self.pushback;
  27.  
  28.     def get_token(self):
  29.         "Get a token from the input stream (or from stack if it's monempty)"
  30.         if self.pushback:
  31.             tok = self.pushback[0]
  32.             self.pushback = self.pushback[1:]
  33.             if (self.debug >= 1):
  34.                 print "Popping " + tok
  35.             return tok
  36.         tok = ''
  37.         while 1:
  38.             nextchar = self.instream.read(1);
  39.             if nextchar == '\n':
  40.                 self.lineno = self.lineno + 1
  41.             if self.debug >= 3:
  42.                 print "In state " + repr(self.state) + " I see character: " + repr(nextchar) 
  43.             if self.state == None:
  44.                 return ''
  45.             elif self.state == ' ':
  46.                 if not nextchar:
  47.                     self.state = None;    # end of file
  48.                     break
  49.                 elif nextchar in self.whitespace:
  50.                     if self.debug >= 2:
  51.                         print "I see whitespace in whitespace state"
  52.                     if self.token:
  53.                         break    # emit current token
  54.                     else:
  55.                         continue
  56.                 elif nextchar in self.commenters:
  57.                     self.instream.readline()
  58.                     self.lineno = self.lineno + 1
  59.                 elif nextchar in self.wordchars:
  60.                     self.token = nextchar
  61.                     self.state = 'a'
  62.                 elif nextchar in self.quotes:
  63.                     self.token = nextchar
  64.                     self.state = nextchar
  65.                 else:
  66.                     self.token = nextchar
  67.                     if self.token:
  68.                         break    # emit current token
  69.                     else:
  70.                         continue
  71.             elif self.state in self.quotes:
  72.                 self.token = self.token + nextchar
  73.                 if nextchar == self.state:
  74.                     self.state = ' '
  75.                     break
  76.             elif self.state == 'a':
  77.                 if not nextchar:
  78.                     self.state = None;    # end of file
  79.                     break
  80.                 elif nextchar in self.whitespace:
  81.                     if self.debug >= 2:
  82.                         print "I see whitespace in word state"
  83.                     self.state = ' '
  84.                     if self.token:
  85.                         break    # emit current token
  86.                     else:
  87.                         continue
  88.                 elif nextchar in self.commenters:
  89.                     self.instream.readline()
  90.                     self.lineno = self.lineno + 1
  91.                 elif nextchar in self.wordchars or nextchar in self.quotes:
  92.                     self.token = self.token + nextchar
  93.                 else:
  94.                     self.pushback = [nextchar] + self.pushback
  95.                     if self.debug >= 2:
  96.                         print "I see punctuation in word state"
  97.                     self.state = ' '
  98.                     if self.token:
  99.                         break    # emit current token
  100.                     else:
  101.                         continue
  102.                 
  103.         result = self.token
  104.         self.token = ''
  105.         if self.debug >= 1:
  106.             print "Token: " + result
  107.         return result
  108.  
  109. if __name__ == '__main__': 
  110.  
  111.     lexer = shlex()
  112.     while 1:
  113.         tt = lexer.get_token()
  114.         if tt != None:
  115.             print "Token: " + repr(tt)
  116.         else:
  117.             break
  118.  
  119.