Commits

Anonymous committed 2f9de5d

[svn] Correct token indices in the raw token lexer.

Comments (0)

Files changed (2)

pygments/lexers/special.py

         return Lexer.get_tokens(self, text)
 
     def get_tokens_unprocessed(self, text):
+        length = 0
         for match in line_re.finditer(text):
             ttypestr, val = match.group().split('\t', 1)
             ttype = _ttype_cache.get(ttypestr)
                     ttype = getattr(ttype, ttype_)
                 _ttype_cache[ttypestr] = ttype
             val = val[1:-2].decode('string-escape')
-            yield 0, ttype, val
+            yield length, ttype, val
+            length += len(val)

pygments/lexers/templates.py

         return rv
 
 
+# Genshi lexers courtesy of Matt Good.
+
 class GenshiTextLexer(RegexLexer):
     name = 'Genshi Text'
     aliases = ['genshitext']