Anonymous avatar Anonymous committed ad6b86c

[svn] Add a new unit test, reduce size of some examples, fix a few errors.

Comments (0)

Files changed (10)

 	@$(PYTHON) scripts/reindent.py -r -B .
 
 test:
-	@$(PYTHON) tests/run.py
+	@$(PYTHON) tests/run.py $(TESTS)

pygments/lexers/__init__.py

     """
     Returns an iterator over all lexer classes.
     """
-    for module_name, name, _, _ in LEXERS.itervalues():
+    for module_name, name, _, _, _ in LEXERS.itervalues():
         if name not in _lexer_cache:
             _load_lexers(module_name)
         yield _lexer_cache[name]

pygments/lexers/special.py

 import cStringIO
 
 from pygments.lexer import Lexer, RegexLexer
-from pygments.token import Token, \
+from pygments.token import Token, Error, \
      Text, Comment, Operator, Keyword, Name, String, Number
 
 
     def get_tokens_unprocessed(self, text):
         length = 0
         for match in line_re.finditer(text):
-            ttypestr, val = match.group().split('\t', 1)
-            ttype = _ttype_cache.get(ttypestr)
-            if not ttype:
-                ttype = Token
-                ttypes = ttypestr.split('.')[1:]
-                for ttype_ in ttypes:
-                    ttype = getattr(ttype, ttype_)
-                _ttype_cache[ttypestr] = ttype
-            val = val[1:-2].decode('string-escape')
+            try:
+                ttypestr, val = match.group().split('\t', 1)
+            except ValueError:
+                val = match.group()
+                ttype = Error 
+            else:
+                ttype = _ttype_cache.get(ttypestr)
+                if not ttype:
+                    ttype = Token
+                    ttypes = ttypestr.split('.')[1:]
+                    for ttype_ in ttypes:
+                        ttype = getattr(ttype, ttype_)
+                    _ttype_cache[ttypestr] = ttype
+                val = val[1:-2].decode('string-escape')
             yield length, ttype, val
             length += len(val)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.