home *** CD-ROM | disk | FTP | other *** search
/ GameStar 2006 January / Gamestar_80_2006-01_dvd.iso / Dema / Civilization4 / data1.cab / Civ4DemoComponent / Assets / Python / System / shlex.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2005-11-09  |  7.4 KB  |  318 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.4)
  3.  
  4. '''A lexical analyzer class for simple shell-like syntaxes.'''
  5. import os.path as os
  6. import sys
  7. from collections import deque
  8.  
  9. try:
  10.     from cStringIO import StringIO
  11. except ImportError:
  12.     from StringIO import StringIO
  13.  
  14. __all__ = [
  15.     'shlex',
  16.     'split']
  17.  
  18. class shlex:
  19.     '''A lexical analyzer class for simple shell-like syntaxes.'''
  20.     
  21.     def __init__(self, instream = None, infile = None, posix = False):
  22.         if isinstance(instream, basestring):
  23.             instream = StringIO(instream)
  24.         
  25.         if instream is not None:
  26.             self.instream = instream
  27.             self.infile = infile
  28.         else:
  29.             self.instream = sys.stdin
  30.             self.infile = None
  31.         self.posix = posix
  32.         if posix:
  33.             self.eof = None
  34.         else:
  35.             self.eof = ''
  36.         self.commenters = '#'
  37.         self.wordchars = 'abcdfeghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
  38.         if self.posix:
  39.             self.wordchars += '\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd8\xd9\xda\xdb\xdc\xdd\xde'
  40.         
  41.         self.whitespace = ' \t\r\n'
  42.         self.whitespace_split = False
  43.         self.quotes = '\'"'
  44.         self.escape = '\\'
  45.         self.escapedquotes = '"'
  46.         self.state = ' '
  47.         self.pushback = deque()
  48.         self.lineno = 1
  49.         self.debug = 0
  50.         self.token = ''
  51.         self.filestack = deque()
  52.         self.source = None
  53.         if self.debug:
  54.             print 'shlex: reading from %s, line %d' % (self.instream, self.lineno)
  55.         
  56.  
  57.     
  58.     def push_token(self, tok):
  59.         '''Push a token onto the stack popped by the get_token method'''
  60.         if self.debug >= 1:
  61.             print 'shlex: pushing token ' + repr(tok)
  62.         
  63.         self.pushback.appendleft(tok)
  64.  
  65.     
  66.     def push_source(self, newstream, newfile = None):
  67.         """Push an input source onto the lexer's input source stack."""
  68.         if isinstance(newstream, basestring):
  69.             newstream = StringIO(newstream)
  70.         
  71.         self.filestack.appendleft((self.infile, self.instream, self.lineno))
  72.         self.infile = newfile
  73.         self.instream = newstream
  74.         self.lineno = 1
  75.         if self.debug:
  76.             if newfile is not None:
  77.                 print 'shlex: pushing to file %s' % (self.infile,)
  78.             else:
  79.                 print 'shlex: pushing to stream %s' % (self.instream,)
  80.         
  81.  
  82.     
  83.     def pop_source(self):
  84.         '''Pop the input source stack.'''
  85.         self.instream.close()
  86.         (self.infile, self.instream, self.lineno) = self.filestack.popleft()
  87.         if self.debug:
  88.             print 'shlex: popping to %s, line %d' % (self.instream, self.lineno)
  89.         
  90.         self.state = ' '
  91.  
  92.     
  93.     def get_token(self):
  94.         """Get a token from the input stream (or from stack if it's nonempty)"""
  95.         if self.pushback:
  96.             tok = self.pushback.popleft()
  97.             if self.debug >= 1:
  98.                 print 'shlex: popping token ' + repr(tok)
  99.             
  100.             return tok
  101.         
  102.         raw = self.read_token()
  103.         if self.source is not None:
  104.             while raw == self.source:
  105.                 spec = self.sourcehook(self.read_token())
  106.                 if spec:
  107.                     (newfile, newstream) = spec
  108.                     self.push_source(newstream, newfile)
  109.                 
  110.                 raw = self.get_token()
  111.         
  112.         while raw == self.eof:
  113.             if not self.filestack:
  114.                 return self.eof
  115.                 continue
  116.             self.pop_source()
  117.             raw = self.get_token()
  118.         if self.debug >= 1:
  119.             if raw != self.eof:
  120.                 print 'shlex: token=' + repr(raw)
  121.             else:
  122.                 print 'shlex: token=EOF'
  123.         
  124.         return raw
  125.  
  126.     
  127.     def read_token(self):
  128.         quoted = False
  129.         escapedstate = ' '
  130.         while True:
  131.             nextchar = self.instream.read(1)
  132.             if nextchar == '\n':
  133.                 self.lineno = self.lineno + 1
  134.             
  135.             if self.debug >= 3:
  136.                 print 'shlex: in state', repr(self.state), 'I see character:', repr(nextchar)
  137.             
  138.             if self.state is None:
  139.                 self.token = ''
  140.                 break
  141.                 continue
  142.             if self.state == ' ':
  143.                 if not nextchar:
  144.                     self.state = None
  145.                     break
  146.                 elif nextchar in self.whitespace:
  147.                     if self.debug >= 2:
  148.                         print 'shlex: I see whitespace in whitespace state'
  149.                     
  150.                     if (self.token or self.posix) and quoted:
  151.                         break
  152.                     
  153.                 elif nextchar in self.commenters:
  154.                     self.instream.readline()
  155.                     self.lineno = self.lineno + 1
  156.                 elif self.posix and nextchar in self.escape:
  157.                     escapedstate = 'a'
  158.                     self.state = nextchar
  159.                 elif nextchar in self.wordchars:
  160.                     self.token = nextchar
  161.                     self.state = 'a'
  162.                 elif nextchar in self.quotes:
  163.                     if not self.posix:
  164.                         self.token = nextchar
  165.                     
  166.                     self.state = nextchar
  167.                 elif self.whitespace_split:
  168.                     self.token = nextchar
  169.                     self.state = 'a'
  170.                 else:
  171.                     self.token = nextchar
  172.                     if (self.token or self.posix) and quoted:
  173.                         break
  174.                     
  175.             nextchar in self.escape
  176.             if self.state in self.quotes:
  177.                 quoted = True
  178.                 if not nextchar:
  179.                     if self.debug >= 2:
  180.                         print 'shlex: I see EOF in quotes state'
  181.                     
  182.                     raise ValueError, 'No closing quotation'
  183.                 
  184.                 if nextchar == self.state:
  185.                     if not self.posix:
  186.                         self.token = self.token + nextchar
  187.                         self.state = ' '
  188.                         break
  189.                     else:
  190.                         self.state = 'a'
  191.                 elif self.posix and nextchar in self.escape and self.state in self.escapedquotes:
  192.                     escapedstate = self.state
  193.                     self.state = nextchar
  194.                 else:
  195.                     self.token = self.token + nextchar
  196.             self.state in self.escapedquotes
  197.             if self.state in self.escape:
  198.                 if not nextchar:
  199.                     if self.debug >= 2:
  200.                         print 'shlex: I see EOF in escape state'
  201.                     
  202.                     raise ValueError, 'No escaped character'
  203.                 
  204.                 if escapedstate in self.quotes and nextchar != self.state and nextchar != escapedstate:
  205.                     self.token = self.token + self.state
  206.                 
  207.                 self.token = self.token + nextchar
  208.                 self.state = escapedstate
  209.                 continue
  210.             if self.state == 'a':
  211.                 if not nextchar:
  212.                     self.state = None
  213.                     break
  214.                 elif nextchar in self.whitespace:
  215.                     if self.debug >= 2:
  216.                         print 'shlex: I see whitespace in word state'
  217.                     
  218.                     self.state = ' '
  219.                     if (self.token or self.posix) and quoted:
  220.                         break
  221.                     
  222.                 elif nextchar in self.commenters:
  223.                     self.instream.readline()
  224.                     self.lineno = self.lineno + 1
  225.                     if self.posix:
  226.                         self.state = ' '
  227.                         if (self.token or self.posix) and quoted:
  228.                             break
  229.                         
  230.                     
  231.                 elif self.posix and nextchar in self.quotes:
  232.                     self.state = nextchar
  233.                 elif self.posix and nextchar in self.escape:
  234.                     escapedstate = 'a'
  235.                     self.state = nextchar
  236.                 elif nextchar in self.wordchars and nextchar in self.quotes or self.whitespace_split:
  237.                     self.token = self.token + nextchar
  238.                 else:
  239.                     self.pushback.appendleft(nextchar)
  240.                     if self.debug >= 2:
  241.                         print 'shlex: I see punctuation in word state'
  242.                     
  243.                     self.state = ' '
  244.                     if self.token:
  245.                         break
  246.                     
  247.             self.whitespace_split
  248.         result = self.token
  249.         self.token = ''
  250.         if self.posix and not quoted and result == '':
  251.             result = None
  252.         
  253.         if self.debug > 1:
  254.             if result:
  255.                 print 'shlex: raw token=' + repr(result)
  256.             else:
  257.                 print 'shlex: raw token=EOF'
  258.         
  259.         return result
  260.  
  261.     
  262.     def sourcehook(self, newfile):
  263.         '''Hook called on a filename to be sourced.'''
  264.         if newfile[0] == '"':
  265.             newfile = newfile[1:-1]
  266.         
  267.         if isinstance(self.infile, basestring) and not os.path.isabs(newfile):
  268.             newfile = os.path.join(os.path.dirname(self.infile), newfile)
  269.         
  270.         return (newfile, open(newfile, 'r'))
  271.  
  272.     
  273.     def error_leader(self, infile = None, lineno = None):
  274.         '''Emit a C-compiler-like, Emacs-friendly error-message leader.'''
  275.         if infile is None:
  276.             infile = self.infile
  277.         
  278.         if lineno is None:
  279.             lineno = self.lineno
  280.         
  281.         return '"%s", line %d: ' % (infile, lineno)
  282.  
  283.     
  284.     def __iter__(self):
  285.         return self
  286.  
  287.     
  288.     def next(self):
  289.         token = self.get_token()
  290.         if token == self.eof:
  291.             raise StopIteration
  292.         
  293.         return token
  294.  
  295.  
  296.  
  297. def split(s, comments = False):
  298.     lex = shlex(s, posix = True)
  299.     lex.whitespace_split = True
  300.     if not comments:
  301.         lex.commenters = ''
  302.     
  303.     return list(lex)
  304.  
  305. if __name__ == '__main__':
  306.     if len(sys.argv) == 1:
  307.         lexer = shlex()
  308.     else:
  309.         file = sys.argv[1]
  310.         lexer = shlex(open(file), file)
  311.     while None:
  312.         tt = lexer.get_token()
  313.         if tt:
  314.             print 'Token: ' + repr(tt)
  315.             continue
  316.         break
  317.  
  318.