Commits

Angus MacArthur committed a8cc640 Merge

updating to newest from upstream

  • Participants
  • Parent commits e0c0fb1, 24c1619

Comments (0)

Files changed (21)

 * Kumar Appaiah -- Debian control lexer
 * Ali Afshar -- image formatter
 * Andreas Amann -- AppleScript lexer
-* Jeffrey Arnold -- R/S, Rd and BUGS lexers
+* Jeffrey Arnold -- R/S, Rd, BUGS, Jags, and Stan lexers
 * Jeremy Ashkenas -- CoffeeScript lexer
 * Stefan Matthias Aust -- Smalltalk lexer
 * Ben Bangert -- Mako lexers
 * Marek Kubica -- Scheme lexer
 * Jochen Kupperschmidt -- Markdown processor
 * Gerd Kurzbach -- Modelica lexer
+* Jon Larimer, Google Inc. -- Smali lexer
 * Olov Lassus -- Dart lexer
 * Sylvestre Ledru -- Scilab lexer
 * Mark Lee -- Vala lexer
 * Steve Spigarelli -- XQuery lexer
 * Jerome St-Louis -- eC lexer
 * James Strachan -- Kotlin lexer
+* Tom Stuart -- Treetop lexer
 * Tiberius Teng -- default style overhaul
 * Jeremy Thurgood -- Erlang, Squid config lexers
 * Erick Tryzelaar -- Felix lexer
 
 Version 1.6
 -----------
-(in development)
+(in development, to be released xx November 2012)
 
 - Lexers added:
 
+  * AspectJ (PR#90)
+  * BUGS-like languages (PR#89)
+  * Ceylon (PR#86)
+  * Croc (new name for MiniD)
+  * CUDA (PR#75)
+  * Dg (PR#116)
+  * Jags (PR#89)
   * Julia (PR#61)
-  * Croc (new name for MiniD)
+  * Kconfig (#711)
+  * Lasso (PR#95)
   * LiveScript (PR#84)
-  * Lasso (PR#95)
-  * BUGS-like languages (PR#89)
-  * Rust (PR#67)
-  * CUDA (PR#75)
-  * Xtend (PR#68)
+  * Monkey (PR#117)
   * Mscgen (PR#80)
-  * Kconfig (#711)
-  * AspectJ (PR#90)
   * Racket (PR#94)
   * Rdoc (PR#99)
+  * Rust (PR#67)
+  * Smali (Dalvik assembly)
+  * SourcePawn (PR#39)
+  * Stan (PR#89)
+  * Treetop (PR#125)
   * VGL (PR#12)
-  * SourcePawn (PR#39)
-  * Ceylon (PR#86)
+  * Windows Registry (#819)
+  * Xtend (PR#68)
 
 - Fix Template Haskell highlighting (PR#63)
 
 - Fix some S/R lexer errors (PR#91)
 
+- Fix a bug in the Prolog lexer with names that start with 'is' (#810)
+
 
 Version 1.5
 -----------

pygments/filters/__init__.py

         Filter.__init__(self, **options)
 
     def filter(self, lexer, stream):
-        output = []
         current_type = None
         current_value = None
         for ttype, value in stream:

pygments/formatters/other.py

                 outfile.write(text.encode())
             flush = outfile.flush
 
-        lasttype = None
-        lastval = u''
         if self.error_color:
             for ttype, value in tokensource:
                 line = "%s\t%r\n" % (ttype, value)

pygments/lexers/_mapping.py

     'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
     'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
     'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
-    'CUDALexer': ('pygments.lexers.compiled', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)),
     'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
     'Cfengine3Lexer': ('pygments.lexers.other', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
     'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire'), (), ('text/html+cheetah', 'text/html+spitfire')),
     'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
     'CommonLispLexer': ('pygments.lexers.functional', 'Common Lisp', ('common-lisp', 'cl'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)),
     'CoqLexer': ('pygments.lexers.functional', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
-    'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx'), ('text/x-c++hdr', 'text/x-c++src')),
+    'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
     'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
     'CrocLexer': ('pygments.lexers.agile', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
     'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
     'CssLexer': ('pygments.lexers.web', 'CSS', ('css',), ('*.css',), ('text/css',)),
     'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)),
     'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)),
+    'CudaLexer': ('pygments.lexers.compiled', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)),
     'CythonLexer': ('pygments.lexers.compiled', 'Cython', ('cython', 'pyx'), ('*.pyx', '*.pxd', '*.pxi'), ('text/x-cython', 'application/x-cython')),
     'DLexer': ('pygments.lexers.compiled', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)),
     'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
     'DartLexer': ('pygments.lexers.web', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
     'DebianControlLexer': ('pygments.lexers.text', 'Debian Control file', ('control',), ('control',), ()),
     'DelphiLexer': ('pygments.lexers.compiled', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)),
+    'DgLexer': ('pygments.lexers.agile', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
     'DiffLexer': ('pygments.lexers.text', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
     'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
     'DtdLexer': ('pygments.lexers.web', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)),
     'ModelicaLexer': ('pygments.lexers.other', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
     'Modula2Lexer': ('pygments.lexers.compiled', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
     'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
+    'MonkeyLexer': ('pygments.lexers.compiled', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
     'MoonScriptLexer': ('pygments.lexers.agile', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
     'MscgenLexer': ('pygments.lexers.other', 'Mscgen', ('mscgen', 'msc'), ('*.msc',), ()),
     'MuPADLexer': ('pygments.lexers.math', 'MuPAD', ('mupad',), ('*.mu',), ()),
     'RdLexer': ('pygments.lexers.math', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)),
     'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3'), ('text/x-rebol',)),
     'RedcodeLexer': ('pygments.lexers.other', 'Redcode', ('redcode',), ('*.cw',), ()),
+    'RegeditLexer': ('pygments.lexers.text', 'reg', (), ('*.reg',), ('text/x-windows-registry',)),
     'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
     'RstLexer': ('pygments.lexers.text', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
     'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
     'SchemeLexer': ('pygments.lexers.functional', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
     'ScilabLexer': ('pygments.lexers.math', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
     'ScssLexer': ('pygments.lexers.web', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
+    'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
     'SmalltalkLexer': ('pygments.lexers.other', 'Smalltalk', ('smalltalk', 'squeak'), ('*.st',), ('text/x-smalltalk',)),
     'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
     'SnobolLexer': ('pygments.lexers.other', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
     'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
     'TexLexer': ('pygments.lexers.text', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
+    'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
     'UrbiscriptLexer': ('pygments.lexers.other', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
     'VGLLexer': ('pygments.lexers.other', 'VGL', ('vgl',), ('*.rpf',), ()),
     'ValaLexer': ('pygments.lexers.compiled', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),

pygments/lexers/_postgres_builtins.py

 
 def parse_datatypes(f):
     dt = set()
-    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
     for line in f:
         if '<sect1' in line:
             break

pygments/lexers/agile.py

 __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
            'Python3Lexer', 'Python3TracebackLexer', 'RubyLexer',
            'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', 'MoonScriptLexer',
-           'CrocLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', 'FactorLexer', 'FancyLexer']
+           'CrocLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', 'FactorLexer',
+           'FancyLexer', 'DgLexer']
 
 # b/w compatibility
 from pygments.lexers.functional import SchemeLexer
              bygroups(Text, using(PythonLexer), Text)),
             (r'^([ \t]*)(\.\.\.)(\n)',
              bygroups(Text, Comment, Text)), # for doctests...
-            (r'^(.+)(: )(.+)(\n)',
+            (r'^([^:]+)(: )(.+)(\n)',
              bygroups(Generic.Error, Text, Name, Text), '#pop'),
             (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)',
              bygroups(Generic.Error, Text), '#pop')
              bygroups(Text, using(Python3Lexer), Text)),
             (r'^([ \t]*)(\.\.\.)(\n)',
              bygroups(Text, Comment, Text)), # for doctests...
-            (r'^(.+)(: )(.+)(\n)',
+            (r'^([^:]+)(: )(.+)(\n)',
              bygroups(Generic.Error, Text, Name, Text), '#pop'),
             (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)',
              bygroups(Generic.Error, Text), '#pop')
             (r":'(\\\\|\\'|[^'])*'", String.Symbol),
             (r"'(\\\\|\\'|[^'])*'", String.Single),
             (r':"', String.Symbol, 'simple-sym'),
+            (r'([a-zA-Z_][a-zA-Z0-9]*)(:)',
+             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
             (r'"', String.Double, 'simple-string'),
             (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
         ]
             (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
             (r'__END__', Comment.Preproc, 'end-part'),
             # multiline regex (after keywords or assignments)
-            (r'(?:^|(?<=[=<>~!])|'
+            (r'(?:^|(?<=[=<>~!:])|'
                  r'(?<=(?:\s|;)when\s)|'
                  r'(?<=(?:\s|;)or\s)|'
                  r'(?<=(?:\s|;)and\s)|'
             (r'\d+', Number.Integer)
         ]
     }
+
+
+class DgLexer(RegexLexer):
+    """
+    Lexer for `dg <http://pyos.github.com/dg>`_,
+    a functional and object-oriented programming language
+    running on the CPython 3 VM.
+
+    *New in Pygments 1.6.*
+    """
+    name = 'dg'
+    aliases = ['dg']
+    filenames = ['*.dg']
+    mimetypes = ['text/x-dg']
+
+    tokens = {
+        'root': [
+            # Whitespace:
+            (r'\s+', Text),
+            (r'#.*?$', Comment.Single),
+            # Lexemes:
+            #  Numbers
+            (r'0[bB][01]+', Number.Bin),
+            (r'0[oO][0-7]+', Number.Oct),
+            (r'0[xX][\da-fA-F]+', Number.Hex),
+            (r'[+-]?\d+\.\d+([eE][+-]?\d+)?[jJ]?', Number.Float),
+            (r'[+-]?\d+[eE][+-]?\d+[jJ]?', Number.Float),
+            (r'[+-]?\d+[jJ]?', Number.Integer),
+            #  Character/String Literals
+            (r"[br]*'''", String, combined('stringescape', 'tsqs', 'string')),
+            (r'[br]*"""', String, combined('stringescape', 'tdqs', 'string')),
+            (r"[br]*'", String, combined('stringescape', 'sqs', 'string')),
+            (r'[br]*"', String, combined('stringescape', 'dqs', 'string')),
+            #  Operators
+            (r"`\w+'*`", Operator), # Infix links
+            #   Reserved infix links
+            (r'\b(or|and|if|unless|else|where|is|in)\b', Operator.Word),
+            (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
+            #  Identifiers
+            #   Python 3 types
+            (r"(?<!\.)(bool|bytearray|bytes|classmethod|complex|dict'?|"
+             r"float|frozenset|int|list'?|memoryview|object|property|range|"
+             r"set'?|slice|staticmethod|str|super|tuple'?|type)"
+             r"(?!['\w])", Name.Builtin),
+            #   Python 3 builtins + some more
+            (r'(?<!\.)(__import__|abs|all|any|bin|bind|chr|cmp|compile|complex|'
+             r'delattr|dir|divmod|drop|dropwhile|enumerate|eval|filter|flip|'
+             r'foldl1?|format|fst|getattr|globals|hasattr|hash|head|hex|id|'
+             r'init|input|isinstance|issubclass|iter|iterate|last|len|locals|'
+             r'map|max|min|next|oct|open|ord|pow|print|repr|reversed|round|'
+             r'setattr|scanl1?|snd|sorted|sum|tail|take|takewhile|vars|zip)'
+             r"(?!['\w])", Name.Builtin),
+            (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
+             Name.Builtin.Pseudo),
+            (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
+             Name.Exception),
+            (r"(?<!\.)(KeyboardInterrupt|SystemExit|StopIteration|"
+             r"GeneratorExit)(?!['\w])", Name.Exception),
+            #   Compiler-defined identifiers
+            (r"(?<![\.\w])(import|inherit|for|while|switch|not|raise|unsafe|"
+             r"yield|with)(?!['\w])", Keyword.Reserved),
+            #   Other links
+            (r"[A-Z_']+\b", Name),
+            (r"[A-Z][\w']*\b", Keyword.Type),
+            (r"\w+'*", Name),
+            #  Blocks
+            (r'[()]', Punctuation),
+        ],
+        'stringescape': [
+            (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
+             r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+        ],
+        'string': [
+            (r'%(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+             '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol),
+            (r'[^\\\'"%\n]+', String),
+            # quotes, percents and backslashes must be parsed one at a time
+            (r'[\'"\\]', String),
+            # unhandled string formatting sign
+            (r'%', String),
+            (r'\n', String)
+        ],
+        'dqs': [
+            (r'"', String, '#pop')
+        ],
+        'sqs': [
+            (r"'", String, '#pop')
+        ],
+        'tdqs': [
+            (r'"""', String, '#pop')
+        ],
+        'tsqs': [
+            (r"'''", String, '#pop')
+        ],
+    }

pygments/lexers/compiled.py

            'DylanLexer', 'ObjectiveCLexer', 'FortranLexer', 'GLShaderLexer',
            'PrologLexer', 'CythonLexer', 'ValaLexer', 'OocLexer', 'GoLexer',
            'FelixLexer', 'AdaLexer', 'Modula2Lexer', 'BlitzMaxLexer',
-           'NimrodLexer', 'FantomLexer', 'RustLexer', 'CUDALexer']
+           'NimrodLexer', 'FantomLexer', 'RustLexer', 'CudaLexer', 'MonkeyLexer']
 
 
 class CLexer(RegexLexer):
     name = 'C++'
     aliases = ['cpp', 'c++']
     filenames = ['*.cpp', '*.hpp', '*.c++', '*.h++',
-                 '*.cc', '*.hh', '*.cxx', '*.hxx']
+                 '*.cc', '*.hh', '*.cxx', '*.hxx',
+                 '*.C', '*.H', '*.cp', '*.CPP']
     mimetypes = ['text/x-c++hdr', 'text/x-c++src']
 
     #: optional Comment or Whitespace
         ],
         'method': [
             include('whitespace'),
+            # TODO unsure if ellipses are allowed elsewhere, see discussion in
+            # Issue 789
+            (r',', Punctuation),
+            (r'\.\.\.', Punctuation),
             (r'(\(.*?\))([a-zA-Z$_][a-zA-Z0-9$_]*)', bygroups(using(this),
                                                               Name.Variable)),
             (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function),
             (r"'(?:''|[^'])*'", String.Atom), # quoted atom
             # Needs to not be followed by an atom.
             #(r'=(?=\s|[a-zA-Z\[])', Operator),
-            (r'(is|<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])',
+            (r'is\b', Operator),
+            (r'(<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])',
              Operator),
             (r'(mod|div|not)\b', Operator),
             (r'_', Keyword), # The don't-care variable
     }
 
 
-class CUDALexer(CLexer):
+class CudaLexer(CLexer):
     """
     For NVIDIA `CUDA™ <http://developer.nvidia.com/category/zone/cuda-zone>`_
     source.
                 elif value in self.functions:
                     token = Name.Function
             yield index, token, value
+
+
+class MonkeyLexer(RegexLexer):
+    """
+    For
+    `Monkey <https://en.wikipedia.org/wiki/Monkey_(programming_language)>`_
+    source code.
+
+    *New in Pygments 1.6.*
+    """
+
+    name = 'Monkey'
+    aliases = ['monkey']
+    filenames = ['*.monkey']
+    mimetypes = ['text/x-monkey'] 
+
+    name_variable = r'[a-z_][a-zA-Z0-9_]*'
+    name_function = r'[A-Z][a-zA-Z0-9_]*'
+    name_constant = r'[A-Z_][A-Z0-9_]*'
+    name_class = r'[A-Z][a-zA-Z0-9_]*'
+    name_module = r'[a-z0-9_]*'
+
+    keyword_type = r'(?:Int|Float|String|Bool|Object|Array|Void)'
+    # ? == Bool // % == Int // # == Float // $ == String
+    keyword_type_special = r'[?%#$]'
+
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            #Text
+            (r'\s+', Text),
+            # Comments
+            (r"'.*", Comment),
+            (r'(?i)^#rem\b', Comment.Multiline, 'comment'),
+            # preprocessor directives
+            (r'(?i)^(?:#If|#ElseIf|#Else|#EndIf|#End|#Print|#Error)\b', Comment.Preproc),
+            # preprocessor variable (any line starting with '#' that is not a directive)
+            (r'^#', Comment.Preproc, 'variables'),
+            # String
+            ('"', String.Double, 'string'),
+            # Numbers
+            (r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
+            (r'\.[0-9]+(?!\.)', Number.Float),
+            (r'[0-9]+', Number.Integer),
+            (r'\$[0-9a-fA-Z]+', Number.Hex),
+            (r'\%[10]+', Number), # Binary
+            # Native data types
+            (r'\b%s\b' % keyword_type, Keyword.Type),
+            # Exception handling
+            (r'(?i)\b(?:Try|Catch|Throw)\b', Keyword.Reserved),
+            (r'Throwable', Name.Exception),
+            # Builtins
+            (r'(?i)\b(?:Null|True|False)\b', Name.Builtin),
+            (r'(?i)\b(?:Self|Super)\b', Name.Builtin.Pseudo),
+            (r'\b(?:HOST|LANG|TARGET|CONFIG)\b', Name.Constant),
+            # Keywords
+            (r'(?i)^(Import)(\s+)(.*)(\n)', bygroups(Keyword.Namespace, Text, Name.Namespace, Text)),
+            (r'(?i)^Strict\b.*\n', Keyword.Reserved),
+            (r'(?i)(Const|Local|Global|Field)(\s+)', bygroups(Keyword.Declaration, Text), 'variables'),
+            (r'(?i)(New|Class|Interface|Extends|Implements)(\s+)', bygroups(Keyword.Reserved, Text), 'classname'),
+            (r'(?i)(Function|Method)(\s+)', bygroups(Keyword.Reserved, Text), 'funcname'),
+            (r'(?i)(?:End|Return|Public|Private|Extern|Property|Final|Abstract)\b', Keyword.Reserved),
+            # Flow Control stuff
+            (r'(?i)(?:If|Then|Else|ElseIf|EndIf|'
+             r'Select|Case|Default|'
+             r'While|Wend|'
+             r'Repeat|Until|Forever|'
+             r'For|To|Until|Step|EachIn|Next|'
+             r'Exit|Continue)\s+', Keyword.Reserved),
+            # not used yet
+            (r'(?i)\b(?:Module|Inline)\b', Keyword.Reserved),
+            # Array
+            (r'[\[\]]', Punctuation),
+            # Other
+            (r'<=|>=|<>|\*=|/=|\+=|-=|&=|~=|\|=|[-&*/^+=<>|~]', Operator),
+            (r'(?i)(?:Not|Mod|Shl|Shr|And|Or)', Operator.Word),
+            (r'[\(\){}!#,.:]', Punctuation),
+            # catch the rest
+            (r'%s\b' % name_constant, Name.Constant),
+            (r'%s\b' % name_function, Name.Function),
+            (r'%s\b' % name_variable, Name.Variable),
+        ],
+        'funcname': [
+            (r'(?i)%s\b' % name_function, Name.Function),
+            (r':', Punctuation, 'classname'),
+            (r'\s+', Text),
+            (r'\(', Punctuation, 'variables'),
+            (r'\)', Punctuation, '#pop') 
+        ],
+        'classname': [
+            (r'%s\.' % name_module, Name.Namespace), 
+            (r'%s\b' % keyword_type, Keyword.Type),
+            (r'%s\b' % name_class, Name.Class),
+            # array (of given size)
+            (r'(\[)(\s*)(\d*)(\s*)(\])',
+             bygroups(Punctuation, Text, Number.Integer, Text, Punctuation)),
+            # generics
+            (r'\s+(?!<)', Text, '#pop'),
+            (r'<', Punctuation, '#push'),
+            (r'>', Punctuation, '#pop'),
+            (r'\n', Text, '#pop'),
+            (r'', Text, '#pop')
+        ],
+        'variables': [
+            (r'%s\b' % name_constant, Name.Constant),
+            (r'%s\b' % name_variable, Name.Variable),
+            (r'%s' % keyword_type_special, Keyword.Type),
+            (r'\s+', Text),
+            (r':', Punctuation, 'classname'),
+            (r',', Punctuation, '#push'),
+            (r'', Text, '#pop')
+        ],
+        'string': [
+            (r'[^"~]+', String.Double),
+            (r'~q|~n|~r|~t|~z|~~', String.Escape),
+            (r'"', String.Double, '#pop'),
+        ],
+        'comment' : [
+            (r'(?i)^#rem.*?', Comment.Multiline, "#push"),
+            (r'(?i)^#end.*?', Comment.Multiline, "#pop"),
+            (r'\n', Comment.Multiline),
+            (r'.+', Comment.Multiline),
+        ],
+    }

pygments/lexers/dalvik.py

+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.dalvik
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexers for Dalvik VM-related languages.
+
+    :copyright: Copyright 2011-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using
+from pygments.token import Keyword, Text, Comment, Name, String, Number, \
+                           Punctuation
+
+__all__ = ['SmaliLexer']
+
+
+class SmaliLexer(RegexLexer):
+    """
+    For `Smali <http://code.google.com/p/smali/>`_ (Android/Dalvik) assembly
+    code.
+
+    *New in Pygments 1.6.*
+    """
+    name = 'Smali'
+    aliases = ['smali']
+    filenames = ['*.smali']
+    mimetypes = ['text/smali']
+
+    tokens = {
+        'root': [
+            include('comment'),
+            include('label'),
+            include('field'),
+            include('method'),
+            include('class'),
+            include('directive'),
+            include('access-modifier'),
+            include('instruction'),
+            include('literal'),
+            include('punctuation'),
+            include('type'),
+            include('whitespace')
+        ],
+        'directive': [
+            (r'^[ \t]*\.(class|super|implements|field|subannotation|annotation|'
+             r'enum|method|registers|locals|array-data|packed-switch|'
+             r'sparse-switch|catchall|catch|line|parameter|local|prologue|'
+             r'epilogue|source)', Keyword),
+            (r'^[ \t]*\.end (field|subannotation|annotation|method|array-data|'
+             'packed-switch|sparse-switch|parameter|local)', Keyword),
+            (r'^[ \t]*\.restart local', Keyword),
+        ],
+        'access-modifier': [
+            (r'(public|private|protected|static|final|synchronized|bridge|'
+             r'varargs|native|abstract|strictfp|synthetic|constructor|'
+             r'declared-synchronized|interface|enum|annotation|volatile|'
+             r'transient)', Keyword),
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+        ],
+        'instruction': [
+            (r'\b[vp]\d+\b', Name.Builtin), # registers
+            (r'\b[a-z][A-Za-z0-9/-]+\s+', Text), # instructions
+        ],
+        'literal': [
+            (r'".*"', String),
+            (r'0x[0-9A-Fa-f]+t?', Number.Hex),
+            (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'[0-9]+L?', Number.Integer),
+        ],
+        'field': [
+            (r'(\$?\b)([A-Za-z0-9_$]*)(:)',
+             bygroups(Punctuation, Name.Variable, Punctuation)),
+        ],
+        'method': [
+            (r'<(?:cl)?init>', Name.Function), # constructor
+            (r'(\$?\b)([A-Za-z0-9_$]*)(\()',
+             bygroups(Punctuation, Name.Function, Punctuation)),
+        ],
+        'label': [
+            (r':[A-Za-z0-9_]+', Name.Label),
+        ],
+        'class': [
+            # class names in the form Lcom/namespace/ClassName;
+            # I only want to color the ClassName part, so the namespace part is
+            # treated as 'Text'
+            (r'(L)((?:[A-Za-z0-9_$]+/)*)([A-Za-z0-9_$]+)(;)',
+                bygroups(Keyword.Type, Text, Name.Class, Text)),
+        ],
+        'punctuation': [
+            (r'->', Punctuation),
+            (r'[{},\(\):=\.-]', Punctuation),
+        ],
+        'type': [
+            (r'[ZBSCIJFDV\[]+', Keyword.Type),
+        ],
+        'comment': [
+            (r'#.*?\n', Comment),
+        ],
+    }

pygments/lexers/functional.py

 
 class RacketLexer(RegexLexer):
     """
-    Lexer for Racket source code.
+    Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly known as
+    PLT Scheme).
 
     *New in Pygments 1.6.*
     """

pygments/lexers/jvm.py

             (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text), 'class'),
             (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
             (r'"(\\\\|\\"|[^"])*"', String),
-            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
             (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)),
             (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label),
             (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
             (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator),
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text)
         ],
             (r'(type)(\s+)', bygroups(Keyword, Text), 'type'),
             (r'""".*?"""(?!")', String),
             (r'"(\\\\|\\"|[^"])*"', String),
-            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
 #            (ur'(\.)(%s|%s|`[^`]+`)' % (idrest, op), bygroups(Operator,
 #             Name.Attribute)),
             (idrest, Name),
             (op, Operator),
             (r'([0-9][0-9]*\.[0-9]*|\.[0-9]+)([eE][+-]?[0-9]+)?[fFdD]?',
              Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text)
         ],
             (r"'(\\\\|\\'|[^'])*'", String.Single),
             (r'\$/((?!/\$).)*/\$', String),
             (r'/(\\\\|\\"|[^/])*/', String),
-            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
             (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)),
             (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label),
             (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
             (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator),
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text)
         ],
             (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
             (r'(isa|[.]{3}|[.]{2}|[=#!<>+-/%&;,.\*\\\(\)\[\]\{\}])', Operator),
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text)
         ],
             (r'(class|interface|object)(\s+)', bygroups(Keyword.Declaration, Text), 'class'),
             (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
             (r'"(\\\\|\\"|[^"])*"', String),
-            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Quoted),
-            (r"`\\.`|`[^\\]`|`\\u[0-9a-f]{4}`", String.Char),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Quoted),
+            (r"`\\.`|`[^\\]`|`\\u[0-9a-fA-F]{4}`", String.Char),
             (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)),
             (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label),
             (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
             (r'\d{1,3}(_\d{3})+\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?', Number.Float),
             (r'[0-9][0-9]*\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float),
             (r'[0-9][0-9]*\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?', Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'\d{1,3}(_\d{3})+[kMGTP]?', Number.Integer),
             (r'[0-9]+[kMGTP]?', Number.Integer),
             (r'\n', Text)
             (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name),
             (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator),
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-f]+', Number.Hex),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text)
         ],

pygments/lexers/math.py

 
 
 class JuliaLexer(RegexLexer):
+    """
+    For `Julia <http://julialang.org/>`_ source code.
+
+    *New in Pygments 1.6.*
+    """
     name = 'Julia'
     aliases = ['julia','jl']
     filenames = ['*.jl']
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
             (r'\d+[eE][+-]?[0-9]+', Number.Float),
             (r'0b[01]+', Number.Binary),
+            (r'0o[0-7]+', Number.Oct),
             (r'0x[a-fA-F0-9]+', Number.Hex),
             (r'\d+', Number.Integer)
         ],
 class JuliaConsoleLexer(Lexer):
     """
     For Julia console sessions. Modeled after MatlabSessionLexer.
+
+    *New in Pygments 1.6.*
     """
     name = 'Julia console'
     aliases = ['jlcon']
 
 class BugsLexer(RegexLexer):
     """
-    Pygments Lexer for OpenBugs and WinBugs models.
+    Pygments Lexer for `OpenBugs <http://www.openbugs.info/w/>`_ and WinBugs
+    models.
 
     *New in Pygments 1.6.*
     """
             include('comments'),
             include('whitespace'),
             # Block start
-            (r'(?s)(model)(\s+)({)',
+            (r'(model)(\s+)({)',
              bygroups(Keyword.Namespace, Text, Punctuation)),
             # Reserved Words
             (r'(for|in)(?![0-9a-zA-Z\._])', Keyword.Reserved),
             include('comments'),
             include('whitespace'),
             # Block start
-            (r'(?s)(model|data)(\s+)({)',
+            (r'(model|data)(\s+)({)',
              bygroups(Keyword.Namespace, Text, Punctuation)),
             (r'var(?![0-9a-zA-Z\._])', Keyword.Declaration),
             # Reserved Words
             # block start
             include('whitespace'),
             # Block start
-            (r'(?s)(%s)(\s*)({)' %
+            (r'(%s)(\s*)({)' %
              r'|'.join(('data', r'transformed\s+?data',
                         'parameters', r'transformed\s+parameters',
                         'model', r'generated\s+quantities')),

pygments/lexers/parsers.py

            'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer',
            #'AntlrCLexer',
            'AntlrCSharpLexer', 'AntlrObjectiveCLexer',
-           'AntlrJavaLexer', "AntlrActionScriptLexer"]
+           'AntlrJavaLexer', "AntlrActionScriptLexer",
+           'TreetopLexer']
 
 
 class RagelLexer(RegexLexer):
     def analyse_text(text):
         return AntlrLexer.analyse_text(text) and \
                re.search(r'^\s*language\s*=\s*ActionScript\s*;', text, re.M)
+
+class TreetopBaseLexer(RegexLexer):
+    """
+    A base lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars.
+    Not for direct use; use TreetopLexer instead.
+
+    *New in Pygments 1.6.*
+    """
+
+    tokens = {
+        'root': [
+            include('space'),
+            (r'require[ \t]+[^\n\r]+[\n\r]', Other),
+            (r'module\b', Keyword.Namespace, 'module'),
+            (r'grammar\b', Keyword, 'grammar'),
+        ],
+        'module': [
+            include('space'),
+            include('end'),
+            (r'module\b', Keyword, '#push'),
+            (r'grammar\b', Keyword, 'grammar'),
+            (r'[A-Z][A-Za-z_0-9]*(?:::[A-Z][A-Za-z_0-9]*)*', Name.Namespace),
+        ],
+        'grammar': [
+            include('space'),
+            include('end'),
+            (r'rule\b', Keyword, 'rule'),
+            (r'include\b', Keyword, 'include'),
+            (r'[A-Z][A-Za-z_0-9]*', Name),
+        ],
+        'include': [
+            include('space'),
+            (r'[A-Z][A-Za-z_0-9]*(?:::[A-Z][A-Za-z_0-9]*)*', Name.Class, '#pop'),
+        ],
+        'rule': [
+            include('space'),
+            include('end'),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'([A-Za-z_][A-Za-z_0-9]*)(:)', bygroups(Name.Label, Punctuation)),
+            (r'[A-Za-z_][A-Za-z_0-9]*', Name),
+            (r'[()]', Punctuation),
+            (r'[?+*/&!~]', Operator),
+            (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex),
+            (r'([0-9]*)(\.\.)([0-9]*)', bygroups(Number.Integer, Operator, Number.Integer)),
+            (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)),
+            (r'{', Punctuation, 'inline_module'),
+            (r'\.', String.Regex),
+        ],
+        'inline_module': [
+            (r'{', Other, 'ruby'),
+            (r'}', Punctuation, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'ruby': [
+            (r'{', Other, '#push'),
+            (r'}', Other, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'space': [
+            (r'[ \t\n\r]+', Whitespace),
+            (r'#[^\n]*', Comment.Single),
+        ],
+        'end': [
+            (r'end\b', Keyword, '#pop'),
+        ],
+    }
+
+class TreetopLexer(DelegatingLexer):
+    """
+    A lexer for `Treetop <http://treetop.rubyforge.org/>`_ grammars.
+
+    *New in Pygments 1.6.*
+    """
+
+    name = 'Treetop'
+    aliases = ['treetop']
+    filenames = ['*.treetop', '*.tt']
+
+    def __init__(self, **options):
+        super(TreetopLexer, self).__init__(RubyLexer, TreetopBaseLexer, **options)

pygments/lexers/templates.py

     `HtmlLexer`.
 
     Nested JavaScript and CSS is also highlighted.
+
+    *New in Pygments 1.6.*
     """
 
     name = 'HTML+Lasso'
     """
     Subclass of the `LassoLexer` which highlights unhandled data with the
     `XmlLexer`.
+
+    *New in Pygments 1.6.*
     """
 
     name = 'XML+Lasso'
     """
     Subclass of the `LassoLexer` which highlights unhandled data with the
     `CssLexer`.
+
+    *New in Pygments 1.6.*
     """
 
     name = 'CSS+Lasso'
     """
     Subclass of the `LassoLexer` which highlights unhandled data with the
     `JavascriptLexer`.
+
+    *New in Pygments 1.6.*
     """
 
     name = 'JavaScript+Lasso'

pygments/lexers/text.py

            'RstLexer', 'VimLexer', 'GettextLexer', 'SquidConfLexer',
            'DebianControlLexer', 'DarcsPatchLexer', 'YamlLexer',
            'LighttpdConfLexer', 'NginxConfLexer', 'CMakeLexer', 'HttpLexer',
-           'PyPyLogLexer']
+           'PyPyLogLexer', 'RegeditLexer']
 
 
 class IniLexer(RegexLexer):
     tokens = {
         'root': [
             (r'\s+', Text),
-            (r'[;#].*?$', Comment),
+            (r'[;#].*', Comment.Single),
             (r'\[.*?\]$', Keyword),
             (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)',
              bygroups(Name.Attribute, Text, Operator, Text, String))
         return text[0] == '[' and text[npos-1] == ']'
 
 
+class RegeditLexer(RegexLexer):
+    """
+    Lexer for `Windows Registry
+    <http://en.wikipedia.org/wiki/Windows_Registry#.REG_files>`_ files produced
+    by regedit.
+
+    *New in Pygments 1.6.*
+    """
+
+    name = 'reg'
+    aliases = []
+    filenames = ['*.reg']
+    mimetypes = ['text/x-windows-registry']
+
+    tokens = {
+        'root': [
+            (r'Windows Registry Editor.*', Text),
+            (r'\s+', Text),
+            (r'[;#].*', Comment.Single),
+            (r'(\[)(-?)(HKEY_[A-Z_]+)(.*?\])$',
+             bygroups(Keyword, Operator, Name.Builtin, Keyword)),
+            # String keys, which obey somewhat normal escaping
+            (r'("(?:\\"|\\\\|[^"])+")([ \t]*)(=)([ \t]*)',
+             bygroups(Name.Attribute, Text, Operator, Text),
+             'value'),
+            # Bare keys (includes @)
+            (r'(.*?)([ \t]*)(=)([ \t]*)',
+             bygroups(Name.Attribute, Text, Operator, Text),
+             'value'),
+        ],
+        'value': [
+            (r'-', Operator, '#pop'), # delete value
+            (r'(dword|hex(?:\([0-9a-fA-F]\))?)(:)([0-9a-fA-F,]+)',
+             bygroups(Name.Variable, Punctuation, Number), '#pop'),
+            # As far as I know, .reg files do not support line continuation.
+            (r'.*', String, '#pop'),
+        ]
+    }
+
+    def analyse_text(text):
+        return text.startswith('Windows Registry Editor')
+
+
 class PropertiesLexer(RegexLexer):
     """
     Lexer for configuration files in Java's properties format.

tests/examplefiles/example.monkey

+Strict
+
+' single line comment
+
+#rem
+multi
+line
+comment
+#end
+
+#rem
+nested
+#rem
+multi
+line
+#end
+comment
+#end
+
+Import mojo
+
+Const ONECONST:Int = 1
+Const TWOCONST := 2
+Const THREECONST := 3, FOURCONST:Int = 4
+
+Global someVariable:Int = 4
+
+' sample class from the documentation
+Class Game Extends App
+
+    Function New()
+    End
+
+    Function DrawSpiral(clock)
+        Local w=DeviceWidth/2
+        For Local i#=0 Until w*1.5 Step .2
+            Local x#,y#
+            x=w+i*Sin(i*3+clock)
+            y=w+i*Cos(i*2+clock)
+            DrawRect  x,y,1,1
+        Next
+        hitbox.Collide(event.pos)
+    End
+
+    Field updateCount
+
+    Method OnCreate()
+        Print "spiral"
+
+        SetUpdateRate 60
+    End
+
+    Method OnUpdate()
+        updateCount+=1
+    End
+
+    Method OnRender()
+        Cls
+        DrawSpiral updateCount
+        DrawSpiral updateCount*1.1
+    End
+
+End
+
+Class Enemy
+  Method Die () Abstract
+End
+
+' extending
+Class Hoodlum Extends Enemy
+    ' field
+    Field testField:Bool = True
+
+    ' naming class with modulepath
+    Local currentNode:list.Node<Vector2D>
+
+    Method Die ()
+        Print "B'oss, he-- he killed me, b'oss!"
+    End
+End
+
+' extending with generics
+Class VectorNode Extends Node<Vector2D>
+End
+
+' interfaces
+Interface Computer
+  Method Boot ()
+  Method Process ()
+  Method Display ()
+End
+
+Class PC Implements Computer
+End
+
+' array syntax
+Global listOfStuff:String[42]
+Global lessStuff:String[5] = listOfStuff[4..8]
+Global oneStuff:String = listOfStuff[23]
+
+'a comma separated sequence
+Global scores:Int[]=[10,20,30]
+'a comma separated sequence
+Global text:String[]=["Hello","There","World"]
+Global worstCase:worst.List<String[]>
+
+' string type
+Global string1:String = "Hello world"
+Global string2$ = "Hello world"
+
+' escape characers in strings
+Global string3 := "Hello~zWorld"
+Global string4 := "~qHello World~q"
+Global string5 := "~tIndented~n"
+Global string6 := "tilda is wavey... ~~"
+
+' string pseudofunctions
+Print "  Hello World  ~n".Trim()    ' prints "Hello World"
+Print "Hello World".ToUpper()       ' prints "HELLO WORLD"
+
+' Boolean shorttype
+Global boolVariable1:Bool = True
+Global boolVariable2? = False
+
+' number formats
+Global hexNum1:Int = $3d0dead
+Global hexNum2% = $CAFEBABE
+
+Global floatNum1:Float = 3.141516
+Global floatNum2# = 3.141516
+Global floatNum3 := .141516
+
+' preprocessor keywords
+#If TARGET = "android"
+DoStuff()
+#ElseIf TARGET = "ios"
+DoOtherStuff()
+#End
+
+' preprocessor variable
+#SOMETHING = True
+#Print SOMETHING
+#If SOMETHING
+#End
+
+' operators
+Global a = 32
+Global b = 32 ~ 0
+b ~= 16
+b |= 16
+b &= 16
+Global c = a | b

tests/examplefiles/example.reg

+Windows Registry Editor Version 5.00
+; comment
+
+[HKEY_CURRENT_USER\SOFTWARE\Pygments]
+@="Hello"
+"Key With Spaces"="Something"
+"Key With ="="With Quotes"
+"Key With = 2"=dword:123
+"Key" = "Value"
+"Hex"=hex(0):1,2,3,a,b,f
+"Hex 2"=hex(5):80,00,00,ff
+
+[-HKEY_CURRENT_USER\SOFTWARE\Pygments\Subkey]
+
+[HKEY_CURRENT_USER\SOFTWARE\Pygments\Subkey2]
+; comment
+@=-
+"Foo"=-
+"Foo"="Value"

tests/examplefiles/hello.smali

+# To Recreate:
+#
+# echo -e 'class hello {\n  public static void main(String[] args) {\n
+# System.out.println("hi");\n  }\n}\n' > hello.java
+# javac -target 1.4 -source 1.4 hello.java
+# dx --dex --output=hello.dex hello.class
+# baksmali hello.dex
+# cat out/hello.smali
+
+.class Lhello;
+.super Ljava/lang/Object;
+.source "hello.java"
+
+
+# direct methods
+.method constructor <init>()V
+    .registers 1
+
+    .prologue
+    .line 1
+    invoke-direct {p0}, Ljava/lang/Object;-><init>()V
+
+    return-void
+.end method
+
+.method public static main([Ljava/lang/String;)V
+    .registers 3
+    .parameter
+
+    .prologue
+    .line 3
+    sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream;
+
+    const-string v1, "hi"
+
+    invoke-virtual {v0, v1}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V
+
+    .line 4
+    return-void
+.end method

tests/examplefiles/inet_pton6.dg

+re  = import!
+sys = import!
+
+
+# IPv6address    =  hexpart [ ":" IPv4address ]
+# IPv4address    =  1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT
+# hexpart        =  [ hexseq ] [ "::" [ hexseq ] ]
+# hexseq         =  hex4 *( ":" hex4)
+# hex4           =  1*4HEXDIG
+hexpart = r'({0}|)(?:::({0}|)|)'.format r'(?:[\da-f]{1,4})(?::[\da-f]{1,4})*'
+addrv4  = r'(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})'
+addrv6  = re.compile $ r'(?i)(?:{})(?::{})?$'.format hexpart addrv4
+
+
+# Parse a base-N number given a list of its digits.
+#
+# :param q: the number of digits in that numeral system
+#
+# :param digits: an iterable of integers in range [0..q]
+#
+# :return: a decimal integer
+#
+base_n = (q digits) -> foldl (x y) -> (x * q + y) 0 digits
+
+
+# Parse a sequence of hexadecimal numbers
+#
+# :param q: a string of colon-separated base-16 integers
+#
+# :return: an iterable of Python ints
+#
+unhex = q -> q and map p -> (int p 16) (q.split ':')
+
+
+# Parse an IPv6 address as specified in RFC 4291.
+#
+# :param address: a string, obviously.
+#
+# :return: an integer which, written in binary form, points to the same node.
+#
+inet_pton6 = address ->
+  raise $ ValueError 'not a valid IPv6 address' unless match = addrv6.match address
+  start, end, *ipv4 = match.groups!
+
+  is_ipv4 = not $ None in ipv4
+  shift   = (7 - start.count ':' - 2 * is_ipv4) * 16
+
+  raise $ ValueError 'not a valid IPv6 address' if (end is None and shift) or shift < 0
+  hexaddr = (base_n 0x10000 (unhex start) << shift) + base_n 0x10000 (unhex $ end or '')
+  hexaddr unless is_ipv4 else (hexaddr << 32) + base_n 0x100 (map int ipv4)
+
+
+inet6_type = q -> switch
+  not q = 'unspecified'
+  q == 1 = 'loopback'
+  (q >>  32)              == 0x000000000000ffff = 'IPv4-mapped'
+  (q >>  64)              == 0xfe80000000000000 = 'link-local'
+  (q >> 120)              != 0x00000000000000ff = 'general unicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000000 = 'multicast w/ reserved scope value'
+  (q >> 112) % (1 << 4)   == 0x000000000000000f = 'multicast w/ reserved scope value'
+  (q >> 112) % (1 << 4)   == 0x0000000000000001 = 'interface-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000004 = 'admin-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000005 = 'site-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000008 = 'organization-local multicast'
+  (q >> 112) % (1 << 4)   == 0x000000000000000e = 'global multicast'
+  (q >> 112) % (1 << 4)   != 0x0000000000000002 = 'multicast w/ unknown scope value'
+  (q >>  24) % (1 << 112) == 0x00000000000001ff = 'solicited-node multicast'
+  True = 'link-local multicast'
+
+
+print $ (x -> (inet6_type x, hex x)) $ inet_pton6 $ sys.stdin.read!.strip!

tests/examplefiles/metagrammar.treetop

+module Treetop
+  module Compiler
+    grammar Metagrammar
+      rule treetop_file
+        requires:(space? require_statement)* prefix:space? module_or_grammar suffix:space? {
+          def compile
+            requires.text_value + prefix.text_value + module_or_grammar.compile + suffix.text_value
+          end
+        }
+      end
+
+      rule require_statement
+        prefix:space? "require" [ \t]+ [^\n\r]+ [\n\r]
+      end
+
+      rule module_or_grammar
+	module_declaration / grammar
+      end
+
+      rule module_declaration
+        prefix:('module' space name:([A-Z] alphanumeric_char* ('::' [A-Z] alphanumeric_char*)*) space) module_contents:(module_declaration / grammar) suffix:(space 'end') {
+          def compile
+            prefix.text_value + module_contents.compile + suffix.text_value
+          end
+
+	  def parser_name
+	    prefix.name.text_value+'::'+module_contents.parser_name
+	  end
+        }
+      end
+
+      rule grammar
+        'grammar' space grammar_name space ('do' space)? declaration_sequence space? 'end' <Grammar>
+      end
+
+      rule grammar_name
+        ([A-Z] alphanumeric_char*)
+      end
+
+      rule declaration_sequence
+        head:declaration tail:(space declaration)* <DeclarationSequence> {
+          def declarations
+            [head] + tail
+          end
+
+          def tail
+            super.elements.map { |elt| elt.declaration }
+          end
+        }
+        /
+        '' {
+          def compile(builder)
+          end
+        }
+      end
+
+      rule declaration
+        parsing_rule / include_declaration
+      end
+
+      rule include_declaration
+        'include' space [A-Z] (alphanumeric_char / '::')* {
+          def compile(builder)
+            builder << text_value
+          end
+        }
+      end
+
+      rule parsing_rule
+        'rule' space nonterminal space ('do' space)? parsing_expression space 'end' <ParsingRule>
+      end
+
+      rule parsing_expression
+        choice / sequence / primary
+      end
+
+      rule choice
+        head:alternative tail:(space? '/' space? alternative)+ <Choice> {
+          def alternatives
+            [head] + tail
+          end
+
+          def tail
+            super.elements.map {|elt| elt.alternative}
+          end
+
+          def inline_modules
+            (alternatives.map {|alt| alt.inline_modules }).flatten
+          end
+        }
+      end
+
+      rule sequence
+        head:labeled_sequence_primary tail:(space labeled_sequence_primary)+ node_class_declarations <Sequence> {
+          def sequence_elements
+            [head] + tail
+          end
+
+          def tail
+            super.elements.map {|elt| elt.labeled_sequence_primary }
+          end
+
+          def inline_modules
+            (sequence_elements.map {|elt| elt.inline_modules}).flatten +
+            [sequence_element_accessor_module] +
+            node_class_declarations.inline_modules
+          end
+
+          def inline_module_name
+            node_class_declarations.inline_module_name
+          end
+        }
+      end
+
+      rule alternative
+        sequence / primary
+      end
+
+      rule primary
+        prefix atomic {
+          def compile(address, builder, parent_expression=nil)
+            prefix.compile(address, builder, self)
+          end
+
+          def prefixed_expression
+            atomic
+          end
+
+          def inline_modules
+            atomic.inline_modules
+          end
+
+          def inline_module_name
+            nil
+          end
+        }
+        /
+	prefix space? predicate_block {
+          def compile(address, builder, parent_expression=nil)
+            prefix.compile(address, builder, self)
+          end
+          def prefixed_expression
+            predicate_block
+          end
+          def inline_modules
+            []
+          end
+        }
+        /
+        atomic suffix node_class_declarations {
+          def compile(address, builder, parent_expression=nil)
+            suffix.compile(address, builder, self)
+          end
+
+          def optional_expression
+            atomic
+          end
+
+          def node_class_name
+            node_class_declarations.node_class_name
+          end
+
+          def inline_modules
+            atomic.inline_modules + node_class_declarations.inline_modules
+          end
+
+          def inline_module_name
+            node_class_declarations.inline_module_name
+          end
+        }
+        /
+        atomic node_class_declarations {
+          def compile(address, builder, parent_expression=nil)
+            atomic.compile(address, builder, self)
+          end
+
+          def node_class_name
+            node_class_declarations.node_class_name
+          end
+
+          def inline_modules
+            atomic.inline_modules + node_class_declarations.inline_modules
+          end
+
+          def inline_module_name
+            node_class_declarations.inline_module_name
+          end
+        }
+      end
+
+      rule labeled_sequence_primary
+        label sequence_primary {
+          def compile(lexical_address, builder)
+            sequence_primary.compile(lexical_address, builder)
+          end
+
+          def inline_modules
+            sequence_primary.inline_modules
+          end
+
+          def label_name
+            if label.name
+              label.name
+            elsif sequence_primary.instance_of?(Nonterminal)
+              sequence_primary.text_value
+            else
+              nil
+            end
+          end
+        }
+      end
+
+      rule label
+        (alpha_char alphanumeric_char*) ':' {
+          def name
+            elements[0].text_value
+          end
+        }
+        /
+        '' {
+          def name
+            nil
+          end
+        }
+      end
+
+      rule sequence_primary
+        prefix atomic {
+          def compile(lexical_address, builder)
+            prefix.compile(lexical_address, builder, self)
+          end
+
+          def prefixed_expression
+            elements[1]
+          end
+
+          def inline_modules
+            atomic.inline_modules
+          end
+
+          def inline_module_name
+            nil
+          end
+        }
+        /
+        prefix space? predicate_block {
+          def compile(address, builder, parent_expression=nil)
+            prefix.compile(address, builder, self)
+          end
+          def prefixed_expression
+            predicate_block
+          end
+          def inline_modules
+            []
+          end
+        }
+        /
+        atomic suffix {
+          def compile(lexical_address, builder)
+            suffix.compile(lexical_address, builder, self)
+          end
+
+          def node_class_name
+            nil
+          end
+
+          def inline_modules
+            atomic.inline_modules
+          end
+
+          def inline_module_name
+            nil
+          end
+        }
+        /
+        atomic
+      end
+
+      rule suffix
+        repetition_suffix / optional_suffix
+      end
+
+      rule optional_suffix
+        '?' <Optional>
+      end
+
+      rule node_class_declarations
+        node_class_expression trailing_inline_module {
+          def node_class_name
+            node_class_expression.node_class_name
+          end
+
+          def inline_modules
+            trailing_inline_module.inline_modules
+          end
+
+          def inline_module
+            trailing_inline_module.inline_module
+          end
+
+          def inline_module_name
+            inline_module.module_name if inline_module
+          end
+        }
+      end
+
+      rule repetition_suffix
+        '+' <OneOrMore> / '*' <ZeroOrMore> / occurrence_range
+      end
+
+      rule occurrence_range
+	space? min:([0-9])* '..' max:([0-9])* <OccurrenceRange>
+      end
+
+      rule prefix
+        '&' <AndPredicate> / '!' <NotPredicate> / '~' <TransientPrefix>
+      end
+
+      rule atomic
+        terminal
+        /
+        nonterminal
+        /
+        parenthesized_expression
+      end
+
+      rule parenthesized_expression
+        '(' space? parsing_expression space? ')' <ParenthesizedExpression> {
+          def inline_modules
+            parsing_expression.inline_modules
+          end
+        }
+      end
+
+      rule nonterminal
+        !keyword_inside_grammar (alpha_char alphanumeric_char*) <Nonterminal>
+      end
+
+      rule terminal
+        quoted_string / character_class / anything_symbol
+      end
+
+      rule quoted_string
+        (single_quoted_string / double_quoted_string) {
+          def string
+            super.text_value
+          end
+        }
+      end
+
+      rule double_quoted_string
+        '"' string:(!'"' ("\\\\" / '\"' / .))* '"' <Terminal>
+      end
+
+      rule single_quoted_string
+        "'" string:(!"'" ("\\\\" / "\\'" / .))* "'" <Terminal>
+      end
+
+      rule character_class
+        '[' characters:(!']' ('\\' . / bracket_expression / !'\\' .))+ ']' <CharacterClass> {
+          def characters
+            super.text_value
+          end
+        }
+      end
+
+      rule bracket_expression
+         '[:' '^'? (
+           'alnum' / 'alpha' / 'blank' / 'cntrl' / 'digit' / 'graph' / 'lower' /
+           'print' / 'punct' / 'space' / 'upper' / 'xdigit' / 'word'
+         ) ':]'
+      end
+
+      rule anything_symbol
+        '.' <AnythingSymbol>
+      end
+
+      rule node_class_expression
+        space '<' (!'>' .)+ '>' {
+          def node_class_name
+            elements[2].text_value
+          end
+        }
+        /
+        '' {
+          def node_class_name
+            nil
+          end
+        }
+      end
+
+      rule trailing_inline_module
+        space inline_module {
+          def inline_modules
+            [inline_module]
+          end
+
+          def inline_module_name
+            inline_module.module_name
+          end
+        }
+        /
+        '' {
+          def inline_modules
+            []
+          end
+
+          def inline_module
+            nil
+          end
+
+          def inline_module_name
+            nil
+          end
+        }
+      end
+
+      rule predicate_block
+        '' inline_module <PredicateBlock>
+      end
+
+      rule inline_module
+        '{' (inline_module / ![{}] .)* '}' <InlineModule>
+      end
+
+      rule keyword_inside_grammar
+        ('rule' / 'end') !non_space_char
+      end
+
+      rule non_space_char
+        !space .
+      end
+
+      rule alpha_char
+        [A-Za-z_]
+      end
+
+      rule alphanumeric_char
+        alpha_char / [0-9]
+      end
+
+      rule space
+        (white / comment_to_eol)+
+      end
+
+      rule comment_to_eol
+        '#' (!"\n" .)*
+      end
+
+      rule white
+        [ \t\n\r]
+      end
+    end
+  end
+end

tests/examplefiles/pytb_test3.pytb

+>>> 3/"3"
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+TypeError: unsupported operand type(s) for /: 'int' and 'str'