1. clach04
  2. jython

Commits

clach04  committed 3d1a0f6

shlex.split() support.
Updated comments on susing Cpython to perform packaging/jar'ing/zipping.
Updated error text for missing files.

  • Participants
  • Parent commits 6d58125
  • Branches 2.2

Comments (0)

Files changed (2)

File lib-python/2.2/shlex.py

View file
+# -*- coding: iso-8859-1 -*-
+# FROM http://hg.python.org/cpython/file/2ce39a7be79d/Lib/shlex.py
+# Lib/shlex.py @ 28703:2ce39a7be79d
 """A lexical analyzer class for simple shell-like syntaxes."""
 
 # Module and documentation by Eric S. Raymond, 21 Dec 1998
 # Input stacking and error message cleanup added by ESR, March 2000
 # push_source() and pop_source() made explicit by ESR, January 2001.
+# Posix compliance, split(), string arguments, and
+# iterator interface by Gustavo Niemeyer, April 2003.
 
 import os.path
 import sys
 
-__all__ = ["shlex"]
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["shlex", "split"]
 
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
-    def __init__(self, instream=None, infile=None):
-        if instream:
+    def __init__(self, instream=None, infile=None, posix=False):
+        if isinstance(instream, basestring):
+            instream = StringIO(instream)
+        if instream is not None:
             self.instream = instream
             self.infile = infile
         else:
             self.instream = sys.stdin
             self.infile = None
+        self.posix = posix
+        if posix:
+            self.eof = None
+        else:
+            self.eof = ''
         self.commenters = '#'
         self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
                           'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
+        if self.posix:
+            self.wordchars += ('��������������������������������'
+                               '������������������������������')
         self.whitespace = ' \t\r\n'
+        self.whitespace_split = False
         self.quotes = '\'"'
+        self.escape = '\\'
+        self.escapedquotes = '"'
         self.state = ' '
         self.pushback = []
         self.lineno = 1
         "Push a token onto the stack popped by the get_token method"
         if self.debug >= 1:
             print "shlex: pushing token " + `tok`
-        self.pushback = [tok] + self.pushback
+        self.pushback.insert(0, tok)
 
     def push_source(self, newstream, newfile=None):
         "Push an input source onto the lexer's input source stack."
+        if isinstance(newstream, basestring):
+            newstream = StringIO(newstream)
         self.filestack.insert(0, (self.infile, self.instream, self.lineno))
         self.infile = newfile
         self.instream = newstream
         self.lineno = 1
         if self.debug:
-            if newfile:
+            if newfile is not None:
                 print 'shlex: pushing to file %s' % (self.infile,)
             else:
                 print 'shlex: pushing to stream %s' % (self.instream,)
     def get_token(self):
         "Get a token from the input stream (or from stack if it's nonempty)"
         if self.pushback:
-            tok = self.pushback[0]
-            self.pushback = self.pushback[1:]
+            tok = self.pushback.pop(0)
             if self.debug >= 1:
                 print "shlex: popping token " + `tok`
             return tok
         # No pushback.  Get a token.
         raw = self.read_token()
         # Handle inclusions
-        while raw == self.source:
-            spec = self.sourcehook(self.read_token())
-            if spec:
-                (newfile, newstream) = spec
-                self.push_source(newstream, newfile)
-            raw = self.get_token()
+        if self.source is not None:
+            while raw == self.source:
+                spec = self.sourcehook(self.read_token())
+                if spec:
+                    (newfile, newstream) = spec
+                    self.push_source(newstream, newfile)
+                raw = self.get_token()
         # Maybe we got EOF instead?
-        while raw == "":
-            if len(self.filestack) == 0:
-                return ""
+        while raw == self.eof:
+            if not self.filestack:
+                return self.eof
             else:
                 self.pop_source()
                 raw = self.get_token()
-         # Neither inclusion nor EOF
+        # Neither inclusion nor EOF
         if self.debug >= 1:
-            if raw:
+            if raw != self.eof:
                 print "shlex: token=" + `raw`
             else:
                 print "shlex: token=EOF"
         return raw
 
     def read_token(self):
-        "Read a token from the input stream (no pushback or inclusions)"
-        while 1:
+        quoted = False
+        escapedstate = ' '
+        while True:
             nextchar = self.instream.read(1)
             if nextchar == '\n':
                 self.lineno = self.lineno + 1
                 elif nextchar in self.whitespace:
                     if self.debug >= 2:
                         print "shlex: I see whitespace in whitespace state"
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
                     self.lineno = self.lineno + 1
+                elif self.posix and nextchar in self.escape:
+                    escapedstate = 'a'
+                    self.state = nextchar
                 elif nextchar in self.wordchars:
                     self.token = nextchar
                     self.state = 'a'
                 elif nextchar in self.quotes:
+                    if not self.posix:
+                        self.token = nextchar
+                    self.state = nextchar
+                elif self.whitespace_split:
                     self.token = nextchar
-                    self.state = nextchar
+                    self.state = 'a'
                 else:
                     self.token = nextchar
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
             elif self.state in self.quotes:
-                self.token = self.token + nextchar
-                if nextchar == self.state:
-                    self.state = ' '
-                    break
-                elif not nextchar:      # end of file
+                quoted = True
+                if not nextchar:      # end of file
                     if self.debug >= 2:
                         print "shlex: I see EOF in quotes state"
                     # XXX what error should be raised here?
                     raise ValueError, "No closing quotation"
+                if nextchar == self.state:
+                    if not self.posix:
+                        self.token = self.token + nextchar
+                        self.state = ' '
+                        break
+                    else:
+                        self.state = 'a'
+                elif self.posix and nextchar in self.escape and \
+                     self.state in self.escapedquotes:
+                    escapedstate = self.state
+                    self.state = nextchar
+                else:
+                    self.token = self.token + nextchar
+            elif self.state in self.escape:
+                if not nextchar:      # end of file
+                    if self.debug >= 2:
+                        print "shlex: I see EOF in escape state"
+                    # XXX what error should be raised here?
+                    raise ValueError, "No escaped character"
+                # In posix shells, only the quote itself or the escape
+                # character may be escaped within quotes.
+                if escapedstate in self.quotes and \
+                   nextchar != self.state and nextchar != escapedstate:
+                    self.token = self.token + self.state
+                self.token = self.token + nextchar
+                self.state = escapedstate
             elif self.state == 'a':
                 if not nextchar:
                     self.state = None   # end of file
                     if self.debug >= 2:
                         print "shlex: I see whitespace in word state"
                     self.state = ' '
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
                     self.lineno = self.lineno + 1
-                elif nextchar in self.wordchars or nextchar in self.quotes:
+                    if self.posix:
+                        self.state = ' '
+                        if self.token or (self.posix and quoted):
+                            break   # emit current token
+                        else:
+                            continue
+                elif self.posix and nextchar in self.quotes:
+                    self.state = nextchar
+                elif self.posix and nextchar in self.escape:
+                    escapedstate = 'a'
+                    self.state = nextchar
+                elif nextchar in self.wordchars or nextchar in self.quotes \
+                    or self.whitespace_split:
                     self.token = self.token + nextchar
                 else:
-                    self.pushback = [nextchar] + self.pushback
+                    self.pushback.insert(0, nextchar)
                     if self.debug >= 2:
                         print "shlex: I see punctuation in word state"
                     self.state = ' '
                         continue
         result = self.token
         self.token = ''
+        if self.posix and not quoted and result == '':
+            result = None
         if self.debug > 1:
             if result:
                 print "shlex: raw token=" + `result`
         if newfile[0] == '"':
             newfile = newfile[1:-1]
         # This implements cpp-like semantics for relative-path inclusion.
-        if type(self.infile) == type("") and not os.path.isabs(newfile):
+        if isinstance(self.infile, basestring) and not os.path.isabs(newfile):
             newfile = os.path.join(os.path.dirname(self.infile), newfile)
         return (newfile, open(newfile, "r"))
 
     def error_leader(self, infile=None, lineno=None):
         "Emit a C-compiler-like, Emacs-friendly error-message leader."
-        if not infile:
+        if infile is None:
             infile = self.infile
-        if not lineno:
+        if lineno is None:
             lineno = self.lineno
         return "\"%s\", line %d: " % (infile, lineno)
 
+    def __iter__(self):
+        return self
+
+    def next(self):
+        token = self.get_token()
+        if token == self.eof:
+            raise StopIteration
+        return token
+
+def split(s, comments=False):
+    lex = shlex(s, posix=True)
+    lex.whitespace_split = True
+    if not comments:
+        lex.commenters = ''
+    return list(lex)
 
 if __name__ == '__main__':
     if len(sys.argv) == 1:

File patch_jython221.py

View file
   * Python 2.3 decimal support
   * Python 2.3 optparse/optik support
 
-NOTE this packaging/patch script currently requires CPython.
+NOTE this packaging/patch script currently requires CPython (2.4 works).
+If Jython 2.2 is used to inject files (e.g for "import decimal"), get error:
+
+    ImportError: error loading from zipfile
+
+using debug:
+
+    echo import decimal | java -jar patched_jython221.jar -Dpython.verbose=debug -
+
+get:
+
+    import: trying source C:\Users\clach04\decimal
+    import: trying precompiled with no sourceC:\Users\clach04\decimal$py.class
+    import: trying source entry: decimal.py from jar/zip file C:\jython2.2.1\patched_jython221.jar\Lib
+    import: loadFromZipFile exception: java.util.zip.ZipException: invalid stored block lengths
+    Traceback (innermost last):
+      File "<stdin>", line 1, in ?
+    ImportError: error loading from zipfile
+
 """
 
 import os
         for nm in os.listdir(path):
             add_to_zip(zf, os.path.join(path, nm), os.path.join(zippath, nm))
     else:
-        raise NotImplementedError('unknown file type')
+        raise NotImplementedError('missing (or unknown file type for) %s' % path)
 
 def add_to_existing_zip(zipfilename, filelist, dest_dir=''):
     if hasattr(zipfile, 'ZIP64_LIMIT'):
     """
     shutil.copy(orig_jar, dest_jar)
     
-    # patch Windows 7 support.
+    # injext updated files.
+    # e.g. Windows 7 support and shlex.split()
     # NOTE this results in duplicate javashell.py entries
     # TODO copy each file from orig zip missing the files to delete/skip
     add_to_existing_zip(dest_jar, [os.path.join('Lib', 'javashell.py')], 'Lib')
+    add_to_existing_zip(dest_jar, [os.path.join('lib-python', '2.2', 'shlex.py')], 'Lib')
     
     # Add back ported modules for convenience
     add_to_existing_zip(dest_jar, [os.path.join('lib-python', 'compat', 'logging')], 'Lib')