Commits

Georg Brandl committed 6faee61 Merge

merge with tim

  • Participants
  • Parent commits 80d62f2, e3898d2

Comments (0)

Files changed (21)

 * Nick Efford -- Python 3 lexer
 * Artem Egorkine -- terminal256 formatter
 * James H. Fisher -- PostScript lexer
+* Naveen Garg - Autohotkey lexer
 * Laurent Gautier -- R/S lexer
 * Krzysiek Goj -- Scala lexer
 * Matt Good -- Genshi, Cheetah lexers
 * Ronny Pfannschmidt -- BBCode lexer
 * Benjamin Peterson -- Test suite refactoring
 * Justin Reidy -- MXML lexer
+* Lubomir Rintel -- GoodData MAQL and CL lexers
 * Andre Roberge -- Tango style
 * Konrad Rudolph -- LaTeX formatter enhancements
 * Mario Ruggier -- Evoque lexers

File pygments/lexer.py

     self.tokens on the first instantiation.
     """
 
+    def _process_regex(cls, regex, rflags):
+        """Preprocess the regular expression component of a token definition."""
+        return re.compile(regex, rflags).match
+
+    def _process_token(cls, token):
+        """Preprocess the token component of a token definition."""
+        assert type(token) is _TokenType or callable(token), \
+               'token type must be simple type or callable, not %r' % (token,)
+        return token
+
+    def _process_new_state(cls, new_state, unprocessed, processed):
+        """Preprocess the state transition action of a token definition."""
+        if isinstance(new_state, str):
+            # an existing state
+            if new_state == '#pop':
+                return -1
+            elif new_state in unprocessed:
+                return (new_state,)
+            elif new_state == '#push':
+                return new_state
+            elif new_state[:5] == '#pop:':
+                return -int(new_state[5:])
+            else:
+                assert False, 'unknown new state %r' % new_state
+        elif isinstance(new_state, combined):
+            # combine a new state from existing ones
+            tmp_state = '_tmp_%d' % cls._tmpname
+            cls._tmpname += 1
+            itokens = []
+            for istate in new_state:
+                assert istate != new_state, 'circular state ref %r' % istate
+                itokens.extend(cls._process_state(unprocessed,
+                                                  processed, istate))
+            processed[tmp_state] = itokens
+            return (tmp_state,)
+        elif isinstance(new_state, tuple):
+            # push more than one state
+            for istate in new_state:
+                assert (istate in unprocessed or
+                        istate in ('#pop', '#push')), \
+                       'unknown new state ' + istate
+            return new_state
+        else:
+            assert False, 'unknown new state def %r' % new_state
+
     def _process_state(cls, unprocessed, processed, state):
+        """Preprocess a single state definition."""
         assert type(state) is str, "wrong state name %r" % state
         assert state[0] != '#', "invalid state name %r" % state
         if state in processed:
             if isinstance(tdef, include):
                 # it's a state reference
                 assert tdef != state, "circular state reference %r" % state
-                tokens.extend(cls._process_state(unprocessed, processed, str(tdef)))
+                tokens.extend(cls._process_state(unprocessed, processed,
+                                                 str(tdef)))
                 continue
 
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
 
             try:
-                rex = re.compile(tdef[0], rflags).match
+                rex = cls._process_regex(tdef[0], rflags)
             except Exception, err:
                 raise ValueError("uncompilable regex %r in state %r of %r: %s" %
                                  (tdef[0], state, cls, err))
 
-            assert type(tdef[1]) is _TokenType or callable(tdef[1]), \
-                   'token type must be simple type or callable, not %r' % (tdef[1],)
+            token = cls._process_token(tdef[1])
 
             if len(tdef) == 2:
                 new_state = None
             else:
-                tdef2 = tdef[2]
-                if isinstance(tdef2, str):
-                    # an existing state
-                    if tdef2 == '#pop':
-                        new_state = -1
-                    elif tdef2 in unprocessed:
-                        new_state = (tdef2,)
-                    elif tdef2 == '#push':
-                        new_state = tdef2
-                    elif tdef2[:5] == '#pop:':
-                        new_state = -int(tdef2[5:])
-                    else:
-                        assert False, 'unknown new state %r' % tdef2
-                elif isinstance(tdef2, combined):
-                    # combine a new state from existing ones
-                    new_state = '_tmp_%d' % cls._tmpname
-                    cls._tmpname += 1
-                    itokens = []
-                    for istate in tdef2:
-                        assert istate != state, 'circular state ref %r' % istate
-                        itokens.extend(cls._process_state(unprocessed,
-                                                          processed, istate))
-                    processed[new_state] = itokens
-                    new_state = (new_state,)
-                elif isinstance(tdef2, tuple):
-                    # push more than one state
-                    for state in tdef2:
-                        assert (state in unprocessed or
-                                state in ('#pop', '#push')), \
-                               'unknown new state ' + state
-                    new_state = tdef2
-                else:
-                    assert False, 'unknown new state def %r' % tdef2
-            tokens.append((rex, tdef[1], new_state))
+                new_state = cls._process_new_state(tdef[2],
+                                                   unprocessed, processed)
+
+            tokens.append((rex, token, new_state))
         return tokens
 
     def process_tokendef(cls, name, tokendefs=None):
+        """Preprocess a dictionary of token definitions."""
         processed = cls._all_tokens[name] = {}
         tokendefs = tokendefs or cls.tokens[name]
         for state in tokendefs.keys():
         return processed
 
     def __call__(cls, *args, **kwds):
+        """Instantiate cls after preprocessing its token definitions."""
         if not hasattr(cls, '_tokens'):
             cls._all_tokens = {}
             cls._tmpname = 0

File pygments/lexers/_mapping.py

     'ApacheConfLexer': ('pygments.lexers.text', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
     'AppleScriptLexer': ('pygments.lexers.other', 'AppleScript', ('applescript',), ('*.applescript',), ()),
     'AsymptoteLexer': ('pygments.lexers.other', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+    'AutohotkeyLexer': ('pygments.lexers.other', 'autohotkey', ('ahk',), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'BBCodeLexer': ('pygments.lexers.text', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
     'BaseMakefileLexer': ('pygments.lexers.text', 'Makefile', ('basemake',), (), ()),
     'BashLexer': ('pygments.lexers.other', 'Bash', ('bash', 'sh', 'ksh'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass'), ('application/x-sh', 'application/x-shellscript')),
     'GherkinLexer': ('pygments.lexers.other', 'Gherkin', ('Cucumber', 'cucumber', 'Gherkin', 'gherkin'), ('*.feature',), ('text/x-gherkin',)),
     'GnuplotLexer': ('pygments.lexers.other', 'Gnuplot', ('gnuplot',), ('*.plot', '*.plt'), ('text/x-gnuplot',)),
     'GoLexer': ('pygments.lexers.compiled', 'Go', ('go',), ('*.go',), ('text/x-gosrc',)),
+    'GoodDataCLLexer': ('pygments.lexers.other', 'GoodData-CL', ('gooddata-cl',), ('*.gdc',), ('text/x-gooddata-cl',)),
     'GroffLexer': ('pygments.lexers.text', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
     'HamlLexer': ('pygments.lexers.web', 'Haml', ('haml', 'HAML'), ('*.haml',), ('text/x-haml',)),
     'HaskellLexer': ('pygments.lexers.functional', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
     'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ('application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako')),
     'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
     'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
+    'MaqlLexer': ('pygments.lexers.other', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
     'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
     'MatlabLexer': ('pygments.lexers.math', 'Matlab', ('matlab', 'octave'), ('*.m',), ('text/matlab',)),
     'MatlabSessionLexer': ('pygments.lexers.math', 'Matlab session', ('matlabsession',), (), ()),
     'SqlLexer': ('pygments.lexers.other', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)),
     'SqliteConsoleLexer': ('pygments.lexers.other', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)),
     'SquidConfLexer': ('pygments.lexers.text', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
+    'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
     'TclLexer': ('pygments.lexers.agile', 'Tcl', ('tcl',), ('*.tcl',), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
     'TcshLexer': ('pygments.lexers.other', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
     'TexLexer': ('pygments.lexers.text', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),

File pygments/lexers/agile.py

         'numbers': [
             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
             (r'\d+[eE][+-]?[0-9]+', Number.Float),
-            (r'0\d+', Number.Oct),
+            (r'0[0-7]+', Number.Oct),
             (r'0[xX][a-fA-F0-9]+', Number.Hex),
             (r'\d+L', Number.Integer.Long),
             (r'\d+', Number.Integer)
             (r"'(\\\\|\\'|[^'])*'", String),
             (r'"(\\\\|\\"|[^"])*"', String),
             (r'`(\\\\|\\`|[^`])*`', String.Backtick),
-            (r'<([^\s>]+)>', String.Regexp),
+            (r'<([^\s>]+)>', String.Regex),
             (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'),
             (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'),
             (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'),

File pygments/lexers/functional.py

         'import': [
             # Import statements
             (r'\s+', Text),
+            (r'"', String, 'string'),
             # after "funclist" state
             (r'\)', Punctuation, '#pop'),
             (r'qualified\b', Keyword),

File pygments/lexers/other.py

 
 from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
      this, do_insertions
-from pygments.token import Error, Punctuation, \
+from pygments.token import Error, Punctuation, Literal, Token, \
      Text, Comment, Operator, Keyword, Name, String, Number, Generic
 from pygments.util import shebang_matches
 from pygments.lexers.web import HtmlLexer
            'MOOCodeLexer', 'SmalltalkLexer', 'TcshLexer', 'LogtalkLexer',
            'GnuplotLexer', 'PovrayLexer', 'AppleScriptLexer',
            'BashSessionLexer', 'ModelicaLexer', 'RebolLexer', 'ABAPLexer',
-           'NewspeakLexer', 'GherkinLexer', 'AsymptoteLexer', 'PostScriptLexer']
+           'NewspeakLexer', 'GherkinLexer', 'AsymptoteLexer',
+           'PostScriptLexer', 'AutohotkeyLexer', 'GoodDataCLLexer',
+           'MaqlLexer']
 
 line_re  = re.compile('.*?\n')
 
             (r'\\[\w\W]', String.Escape),
             (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
             (r'[\[\]{}()=]', Operator),
-            (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+            (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
             (r'&&|\|\|', Operator),
         ],
         'data': [
             (r'([0-8]{3}|n|r|t|b|f|\\|\(|\)|)', String.Escape, '#pop'),
         ],
     }
+
+
+class AutohotkeyLexer(RegexLexer):
+    """
+    For `autohotkey <http://www.autohotkey.com/>`_ source code.
+
+    *New in Pygments 1.4.*
+    """
+    name = 'autohotkey'
+    aliases = ['ahk']
+    filenames = ['*.ahk', '*.ahkl']
+    mimetypes = ['text/x-autohotkey']
+
+    flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (r'^\(', String, 'continuation'),
+            include('comments'),
+            (r'(^\s*)(\w+)(\s*)(=)',
+             bygroups(Text.Whitespace, Name, Text.Whitespace, Operator),
+             'command'),
+            (r'([\w#@$?\[\]]+)(\s*)(\()',
+             bygroups(Name.Function, Text.Whitespace, Punctuation),
+             'parameters'),
+            include('directives'),
+            include('labels'),
+            include('commands'),
+            include('expressions'),
+            include('numbers'),
+            include('literals'),
+            include('keynames'),
+            include('keywords'),
+        ],
+        'command': [
+            include('comments'),
+            include('whitespace'),
+            (r'^\(', String, 'continuation'),
+            (r'[^\n]*?(?=;*|$)', String, '#pop'),
+            include('numbers'),
+            include('literals'),
+        ],
+
+        'expressions': [
+            include('comments'),
+            include('whitespace'),
+            include('numbers'),
+            include('literals'),
+            (r'([]\w#@$?[]+)(\s*)(\()',
+             bygroups(Name.Function, Text.Whitespace, Punctuation),
+             'parameters'),
+            (r'A_\w+', Name.Builtin),
+            (r'%[]\w#@$?[]+?%', Name.Variable),
+            # blocks: if, else, function definitions
+            (r'{', Punctuation, 'block'),
+            # parameters in function calls
+            ],
+        'literals': [
+            (r'"', String, 'string'),
+            (r'A_\w+', Name.Builtin),
+            (r'%[]\w#@$?[]+?%', Name.Variable),
+            (r'[-~!%^&*+|?:<>/=]=?', Operator, 'expressions'),
+            (r'==', Operator, 'expressions'),
+            ('[{()},.%#`;]', Punctuation),
+            (r'\\', Punctuation),
+            include('keywords'),
+            (r'\w+', Text),
+            ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'""|`.', String.Escape),
+            (r'[^\`"\n]+', String), # all other characters
+        ],
+        'block': [
+            include('root'),
+            ('{', Punctuation, '#push'),
+            ('}', Punctuation, '#pop'),
+        ],
+        'parameters': [
+            (r'\)', Punctuation, '#pop'),
+            (r'\(', Punctuation, '#push'),
+            include('numbers'),
+            include('literals'),
+            include('whitespace'),
+        ],
+        'keywords': [
+            (r'(static|global|local)\b', Keyword.Type),
+            (r'(if|else|and|or)\b', Keyword.Reserved),
+            ],
+        'directives': [
+            (r'#\w+?\s', Keyword),
+            ],
+        'labels': [
+            # hotkeys and labels
+            # technically, hotkey names are limited to named keys and buttons
+            (r'(^\s*)([^:\s]+?:{1,2})', bygroups(Text.Whitespace, Name.Label)),
+             # hotstrings
+            (r'(^\s*)(::[]\w#@$?[]+?::)', bygroups(Text.Whitespace, Name.Label)),
+            ],
+        'comments': [
+            (r'^;+.*?$', Comment.Single),  # beginning of line comments
+            (r'(?<=\s);+.*?$', Comment.Single),    # end of line comments
+            (r'^/\*.*?\n\*/', Comment.Multiline),
+            (r'(?<!\n)/\*.*?\n\*/', Error),  # must be at start of line
+            ],
+        'whitespace': [
+            (r'[ \t]+', Text.Whitespace),
+            ],
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'0[0-7]+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+L', Number.Integer.Long),
+            (r'\d+', Number.Integer)
+        ],
+        'continuation': [
+            (r'\n\)', Punctuation, '#pop'),
+            (r'\s[^\n\)]+', String),
+        ],
+        'keynames': [
+            (r'\[[^\]]+\]', Keyword, 'keynames')
+        ],
+        'commands': [
+            (r'(autotrim|blockinput|break|click|'
+             r'clipwait|continue|control|'
+             r'controlclick|controlfocus|controlget|'
+             r'controlgetfocus|controlgetpos|controlgettext|'
+             r'controlmove|controlsend|controlsendraw|'
+             r'controlsettext|coordmode|critical|'
+             r'detecthiddentext|detecthiddenwindows|'
+             r'dllcall|drive|'
+             r'driveget|drivespacefree|'
+             r'else|envadd|envdiv|'
+             r'envget|envmult|envset|'
+             r'envsub|envupdate|exit|'
+             r'exitapp|fileappend|filecopy|'
+             r'filecopydir|filecreatedir|filecreateshortcut|'
+             r'filedelete|filegetattrib|filegetshortcut|'
+             r'filegetsize|filegettime|filegetversion|'
+             r'fileinstall|filemove|filemovedir|'
+             r'fileread|filereadline|filerecycle|'
+             r'filerecycleempty|fileremovedir|fileselectfile|'
+             r'fileselectfolder|filesetattrib|filesettime|'
+             r'formattime|gosub|'
+             r'goto|groupactivate|groupadd|'
+             r'groupclose|groupdeactivate|gui|'
+             r'guicontrol|guicontrolget|hotkey|'
+             r'ifexist|ifgreater|ifgreaterorequal|'
+             r'ifinstring|ifless|iflessorequal|'
+             r'ifmsgbox|ifnotequal|ifnotexist|'
+             r'ifnotinstring|ifwinactive|ifwinexist|'
+             r'ifwinnotactive|ifwinnotexist|imagesearch|'
+             r'inidelete|iniread|iniwrite|'
+             r'input|inputbox|keyhistory|'
+             r'keywait|listhotkeys|listlines|'
+             r'listvars|loop|'
+             r'menu|mouseclick|mouseclickdrag|'
+             r'mousegetpos|mousemove|msgbox|'
+             r'onmessage|onexit|outputdebug|'
+             r'pixelgetcolor|pixelsearch|postmessage|'
+             r'process|progress|random|'
+             r'regexmatch|regexreplace|registercallback|'
+             r'regdelete|regread|regwrite|'
+             r'reload|repeat|return|'
+             r'run|runas|runwait|'
+             r'send|sendevent|sendinput|'
+             r'sendmessage|sendmode|sendplay|'
+             r'sendraw|setbatchlines|setcapslockstate|'
+             r'setcontroldelay|setdefaultmousespeed|setenv|'
+             r'setformat|setkeydelay|setmousedelay|'
+             r'setnumlockstate|setscrolllockstate|'
+             r'setstorecapslockmode|'
+             r'settimer|settitlematchmode|setwindelay|'
+             r'setworkingdir|shutdown|sleep|'
+             r'sort|soundbeep|soundget|'
+             r'soundgetwavevolume|soundplay|soundset|'
+             r'soundsetwavevolume|splashimage|splashtextoff|'
+             r'splashtexton|splitpath|statusbargettext|'
+             r'statusbarwait|stringcasesense|stringgetpos|'
+             r'stringleft|stringlen|stringlower|'
+             r'stringmid|stringreplace|stringright|'
+             r'stringsplit|stringtrimleft|stringtrimright|'
+             r'stringupper|suspend|sysget|'
+             r'thread|tooltip|transform|'
+             r'traytip|urldownloadtofile|while|'
+             r'varsetcapacity|'
+             r'winactivate|winactivatebottom|winclose|'
+             r'winget|wingetactivestats|wingetactivetitle|'
+             r'wingetclass|wingetpos|wingettext|'
+             r'wingettitle|winhide|winkill|'
+             r'winmaximize|winmenuselectitem|winminimize|'
+             r'winminimizeall|winminimizeallundo|winmove|'
+             r'winrestore|winset|winsettitle|'
+             r'winshow|winwait|winwaitactive|'
+             r'winwaitclose|winwaitnotactive'
+             r'true|false|NULL)\b', Keyword, 'command'),
+            ],
+
+        }
+
+class MaqlLexer(RegexLexer):
+    """
+    Lexer for `GoodData MAQL <https://secure.gooddata.com/docs/html/advanced.metric.tutorial.html>`_
+    scripts.
+
+    *New in Pygments 1.4.*
+    """
+
+    name = 'MAQL'
+    aliases = ['maql']
+    filenames = ['*.maql']
+    mimetypes = ['text/x-gooddata-maql','application/x-gooddata-maql']
+
+    flags = re.IGNORECASE
+    tokens = {
+        'root': [
+            # IDENTITY
+            (r'IDENTIFIER\b', Name.Builtin),
+            # IDENTIFIER
+            (r'\{[^}]+\}', Name.Variable),
+            # NUMBER
+            (r'[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]{1,3})?', Literal.Number),
+            # STRING
+            (r'"', Literal.String, 'string-literal'),
+            #  RELATION
+            (r'\<\>|\!\=', Operator),
+            (r'\=|\>\=|\>|\<\=|\<', Operator),
+            # :=
+            (r'\:\=', Operator),
+            # OBJECT
+            (r'\[[^]]+\]', Name.Variable.Class),
+            # keywords
+            (r'(DIMENSIONS?|BOTTOM|METRIC|COUNT|OTHER|FACT|WITH|TOP|OR|'
+             r'ATTRIBUTE|CREATE|PARENT|FALSE|ROWS?|FROM|ALL|AS|PF|'
+             r'COLUMNS?|DEFINE|REPORT|LIMIT|TABLE|LIKE|AND|BY|'
+             r'BETWEEN|EXCEPT|SELECT|MATCH|WHERE|TRUE|FOR|IN|'
+             r'WITHOUT|FILTER|ALIAS|ORDER|FACT|WHEN|NOT|ON|'
+             r'KEYS|KEY|FULLSET|PRIMARY|LABELS|LABEL|VISUAL|'
+             r'TITLE|DESCRIPTION|FOLDER|ALTER|DROP|ADD|DATASET|'
+             r'DATATYPE|INT|BIGINT|DOUBLE|DATE|VARCHAR|DECIMAL|'
+             r'SYNCHRONIZE|TYPE|DEFAULT|ORDER|ASC|DESC|HYPERLINK|'
+             r'INCLUDE|TEMPLATE|MODIFY)\b', Keyword),
+            # FUNCNAME
+            (r'[a-zA-Z]\w*\b', Name.Function),
+            # Comments
+            (r'#.*', Comment.Single),
+            # Punctuation
+            (r'[,;\(\)]', Token.Punctuation),
+            # Space is not significant
+            (r'\s+', Text)
+        ],
+        'string-literal': [
+            (r'\\[tnrfbae"\\]', String.Escape),
+            (r'"', Literal.String, '#pop'),
+            (r'[^\\"]+', Literal.String)
+        ]
+    }
+
+class GoodDataCLLexer(RegexLexer):
+    """
+    Lexer for `GoodData-CL <http://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/com/gooddata/processor/COMMANDS.txt>`_
+    script files.
+
+    *New in Pygments 1.4.*
+    """
+
+    name = 'GoodData-CL'
+    aliases = ['gooddata-cl']
+    filenames = ['*.gdc']
+    mimetypes = ['text/x-gooddata-cl']
+
+    flags = re.IGNORECASE
+    tokens = {
+        'root': [
+            # Comments
+            (r'#.*', Comment.Single),
+            # Function call
+            (r'[a-zA-Z]\w*', Name.Function),
+            # Argument list
+            (r'\(', Token.Punctuation, 'args-list'),
+            # Punctuation
+            (r';', Token.Punctuation),
+            # Space is not significant
+            (r'\s+', Text)
+        ],
+        'args-list': [
+            (r'\)', Token.Punctuation, '#pop'),
+            (r',', Token.Punctuation),
+            (r'[a-zA-Z]\w*', Name.Variable),
+            (r'=', Operator),
+            (r'"', Literal.String, 'string-literal'),
+            (r'[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]{1,3})?', Literal.Number),
+            # Space is not significant
+            (r'\s', Text)
+        ],
+        'string-literal': [
+            (r'\\[tnrfbae"\\]', String.Escape),
+            (r'"', Literal.String, '#pop'),
+            (r'[^\\"]+', Literal.String)
+        ]
+    }

File pygments/lexers/templates.py

            'CheetahXmlLexer', 'CheetahJavascriptLexer',
            'EvoqueLexer', 'EvoqueHtmlLexer', 'EvoqueXmlLexer',
            'ColdfusionLexer', 'ColdfusionHtmlLexer',
-           'VelocityLexer','VelocityHtmlLexer','VelocityXmlLexer']
+           'VelocityLexer', 'VelocityHtmlLexer', 'VelocityXmlLexer',
+           'SspLexer']
 
 
 class ErbLexer(Lexer):
     flags = re.MULTILINE | re.DOTALL
 
     identifier = r'[a-zA-Z_][a-zA-Z0-9_]*'
-    
+
     tokens = {
         'root': [
             (r'[^{#$]+', Other),
 
     def __init__(self, **options):
         super(VelocityXmlLexer, self).__init__(XmlLexer, VelocityLexer,
-                                             **options)
-    
+                                               **options)
+
     def analyse_text(text):
         rv = VelocityLexer.analyse_text(text) - 0.01
         if looks_like_xml(text):
              r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b',
              Keyword),
             (r'(loop|block|super|forloop)\b', Name.Builtin),
-            (r'[a-zA-Z][a-zA-Z0-9_]*', Name.Variable),
+            (r'[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable),
             (r'\.[a-zA-Z0-9_]+', Name.Variable),
             (r':?"(\\\\|\\"|[^"])*"', String.Double),
             (r":?'(\\\\|\\'|[^'])*'", String.Single),
         super(ColdfusionHtmlLexer, self).__init__(HtmlLexer, ColdfusionMarkupLexer,
                                                   **options)
 
+
+class SspLexer(DelegatingLexer):
+    """
+    Lexer for Scalate Server Pages.
+
+    *New in Pygments 1.4.*
+    """
+    name = 'Scalate Server Page'
+    aliases = ['ssp']
+    filenames = ['*.ssp']
+    mimetypes = ['application/x-ssp']
+
+    def __init__(self, **options):
+        super(SspLexer, self).__init__(XmlLexer, JspRootLexer, **options)
+
+    def analyse_text(text):
+        rv = 0.0
+        if re.search('val \w+\s*:', text):
+            rv += 0.6
+        if looks_like_xml(text):
+            rv += 0.2
+        if '<%' in text and '%>' in text:
+            rv += 0.1
+        return rv

File pygments/lexers/text.py

             (r'\s+', Text),
             (r'[;#].*?$', Comment),
             (r'\[.*?\]$', Keyword),
-            (r'(.*?)([ \t]*)(=)([ \t]*)(.*?)$',
+            (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)',
              bygroups(Name.Attribute, Text, Operator, Text, String))
         ]
     }

File pygments/lexers/web.py

             (r'\$\{\$+[a-zA-Z_][a-zA-Z0-9_]*\}', Name.Variable),
             (r'\$+[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable),
             (r'[\\a-zA-Z_][\\a-zA-Z0-9_]*', Name.Other),
-            (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|"
-             r"0[xX][0-9a-fA-F]+[Ll]?", Number),
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'0[0-7]+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
             (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
             (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
             (r'"', String.Double, 'string'),
 
 class ScamlLexer(ExtendedRegexLexer):
     """
-    For Scaml markup.
-    Scaml is Haml for Scala, see:
-    http://scalate.fusesource.org/documentation/scaml-reference.html#features
+    For `Scaml markup <http://scalate.fusesource.org/>`_.  Scaml is Haml for Scala.
 
     *New in Pygments 1.4.*
     """
             (qname, Name.Variable, 'operator'),
         ]
     }
+

File pygments/styles/__init__.py

     'native':   'native::NativeStyle',
     'fruity':   'fruity::FruityStyle',
     'bw':       'bw::BlackWhiteStyle',
+    'vim':      'vim::VimStyle',
     'vs':       'vs::VisualStudioStyle',
     'tango':    'tango::TangoStyle',
 }

File scripts/vim2pygments.py

         if bg_color:
             out.write('    background_color = %r\n' % bg_color[0])
         out.write('    styles = {\n')
-        out.write('        %-20s%r\n' % ('Token:', default_token))
+        out.write('        %-20s%r,\n' % ('Token:', default_token))
         for token, definition in tokens:
             if definition:
-                out.write('        %-20s%r\n' % (token + ':', definition))
+                out.write('        %-20s%r,\n' % (token + ':', definition))
         out.write('    }')
 
     def __repr__(self):

File tests/examplefiles/demo.ahk

+/*
+multiline comment
+msgbox(comment)
+*/
+send, key[pgdn]
+string := "hello" . x . "world!"
+string := "hello ""world""! "
+string := "hello `"world""! "
+; single line comment1
+;;;  single line comment2
+
+::stopi::viper_off()  
+
+a::send, ^a
+mylabel:send, ^{space}  ;; set mark
+e::send, ^e
+n::
+  send, ^n
+  return
+!i::     
+viper("iviper")   ; stdlib
+x = "viper"" "  ; escaped quote
+Return
+#If WinExist("iviper" )
+indexdir = %A_ScriptDir%\%dir%
+FileCreateDir, % indexdir
+fileindex = %indexdir%\_files
+FileSelectFile, file,,, Select an image:, Images (*.gif; *.jpg; *.bmp; *.png; *.tif; *.ico; *.cur; *.ani; *.exe; *.dll)
+
+; viper
+
+
+i::viper_off() 
+#If
+
+;; keybindings
+#If WinExist("iviper") and WinActive("ahk_class Emacs")
+
+p::
+k::
+send, ^p
+return
+,::send, +!,  ;; beginning of page
+.::send, +!.  ;; end of page
+[::send, !a
+]::send, !e  
+d:: ^k  ;; kill line
+x:: send ^d
+\:: ^!k  ;; kill next word or sexp
+
+
+#IfWinActive
+#Persistent
+  
+F2::     ;; hotkey
+start:   ;; label
+start2:   ; label
+  ppm := ppm_new(50, 50, 255)    
+  ppm_fill(ppm, 80, 90, 95)      
+  msgbox % getPixel(ppm, 1, 1)    
+  setPixel(90, 90, 90, ppm, 1, 1)
+  msgbox % getPixel(ppm, 1, 1)
+  ListVars  ; command
+  msgbox % ppm
+  return
+  
+  
+  ppm_read(file)
+  {
+	fileread, ppm, % file
+ return ppm
+}
+
+::hotstring::
+::hot3::
+ppm_width(ppm)
+{
+ regexmatch(ppm, "\R(\d+)\s(\d+)", dim)
+ return dim1
+}
+ppm_height(ppm)
+{
+ regexmatch(ppm, "\R(\d+)\s(\d+)", dim)
+    return dim2
+} 
+
+ppm_colors(ppm)
+{
+regexmatch(ppm, "\R(\d+)\D*\R", colors)  ; \R stands for any
+return colors1
+}
+
+ppm_data(ppm)
+{
+pos :=  regexmatch(ppm, "\R(\d+)\D*\R", colors)  ; \R stands for any newline
+stringtrimleft, data, ppm, pos + strlen(colors1)
+return data
+}
+ppm_header(ppm)
+{
+pos :=  regexmatch(ppm, "\R(\d+)\D*\R", colors)  ; \R stands for any newline
+stringleft, header, ppm, pos + strlen(colors1)
+return header
+}
+
+ppm_fill(ByRef ppm, r, g, b)
+{
+  width := ppm_width(ppm)	
+  height := ppm_height(ppm)
+  header := ppm_header(ppm)
+  headerLength := strlen(header) 
+  varsetcapacity(data, width * height, 0)
+  loop, % (width * height)
+  {
+	if r
+   numput(r, data, (A_Index - 1) * 3, "uchar")
+ if g
+   numput(g, data, (A_Index - 1) * 3 + 1, "uchar")
+ if b
+   numput(b, data, (A_Index - 1) * 3 + 2, "uchar")
+}
+VarCopy(&ppm + headerLength, &data, width * height)
+
+}
+
+ppm_new(width, height, colors)
+{
+  header = P6`n%width% %height%`n%colors%`n
+  headerLength := strlen(header)
+  varsetcapacity(ppm, width * height + headerLength, 1)
+  varsetcapacity(data, width * height, 0)
+  VarCopy(&ppm, &header, headerLength)
+  VarCopy(&ppm + headerLength, &data, width * height)
+  return ppm
+}
+
+heredoc = 
+(
+  P6
+  # lasdjkf
+  87 09
+  255
+  color data...
+)
+
+; Example: Simple image viewer:
+
+Gui, +Resize
+Gui, Add, Button, default, &Load New Image
+Gui, Add, Radio, ym+5 x+10 vRadio checked, Load &actual size
+Gui, Add, Radio, ym+5 x+10, Load to &fit screen
+Gui, Add, Pic, xm vPic
+Gui, Show
+return
+
+ButtonLoadNewImage:
+FileSelectFile, file,,, Select an image:, Images (*.gif; *.jpg; *.bmp; *.png; *.tif; *.ico; *.cur; *.ani; *.exe; *.dll)
+if file =
+    return
+Gui, Submit, NoHide ; Save the values of the radio buttons.
+if Radio = 1  ; Display image at its actual size.
+{
+    Width = 0
+    Height = 0
+}
+else ; Second radio is selected: Resize the image to fit the screen.
+{
+    Width := A_ScreenWidth - 28  ; Minus 28 to allow room for borders and margins inside.
+    Height = -1  ; "Keep aspect ratio" seems best.
+}
+GuiControl,, Pic, *w%width% *h%height% %file%  ; Load the image.
+Gui, Show, xCenter y0 AutoSize, %file%  ; Resize the window to match the picture size.
+return
+
+GuiClose:
+ExitApp
+; Example: Simple text editor with menu bar.
+
+; Create the sub-menus for the menu bar:
+Menu, FileMenu, Add, &New, FileNew
+

File tests/examplefiles/import.hs

+import "mtl" Control.Monad.Trans
+
+main :: IO ()
+main = putStrLn "hello world"

File tests/examplefiles/ints.php

+<?php
+
+$x = array(0123, 0x0a, -0x0a, -0123, 12, -12);
+
+?>
+<?php
+$a = 1.234; 
+$b = 1.2e3; 
+$c = 7E-10;
+?>

File tests/examplefiles/stripheredoc.sh

+cat <<-EOF
+  Hello world $PATH
+  EOF

File tests/examplefiles/test.gdc

+# Execute the date dimension MAQL script
+ExecuteMaql(maqlFile="examples/quotes/quote_date.maql");
+
+# load the stock quotes data file
+# the data file config has been generated
+LoadCsv(csvDataFile="examples/quotes/quotes.csv",
+	header="true",
+	configFile="examples/quotes/quotes.config.xml");
+
+# transfer the stock quotes data
+TransferLastSnapshot();
+
+LoadGoogleAnalytics(configFile="examples/ga/ga.config.xml",username="example@gmail.com",password="******",profileId="ga:7468896",dimensions="ga:date|ga:browser|ga:browserVersion|ga:country|ga:isMobile",metrics="ga:bounces|ga:newVisits|ga:pageViews|ga:visits",startDate="2008-01-01",endDate="2010-06-15");

File tests/examplefiles/test.ini

+[section]
+
+foo = bar
+continued = foo
+  baz
+conttwo =
+  foo
+; comment
+# comment
+

File tests/examplefiles/test.maql

+# MAQL script
+CREATE DATASET {dataset.quotes} VISUAL(TITLE "St\\tock Qu\totes Data");
+
+# A comment
+CREATE DATASET {dataset.quotes} VISUAL(TITLE "Stock Qu\"otes Data");
+
+CREATE DATASET {dataset.quotes} VISUAL(TITLE "Stock Quotes Data");
+
+ALTER DATASET {dataset.quotes} ADD {attribute.sector};
+
+ALTER DATASET {dataset.quotes} DROP {attribute.symbol};
+
+ALTER DATASET {dataset.quotes} VISUAL(TITLE "Internal Quotes Data");
+
+CREATE ATTRIBUTE {attr.quotes.symbol}
+    VISUAL(TITLE "Symbol", FOLDER {folder.quotes.attr})
+    AS {d_quotes_symbol.nm_symbol};
+
+ALTER ATTRIBUTE {attr.quotes.symbol}
+    ADD LABELS {attr.quotes.company} VISUAL(TITLE "Company")
+    AS {d_quotes_symbol.nm_company};
+
+CREATE FACT {fact.quotes.open_price}
+    VISUAL( TITLE "Open Price", FOLDER {folder.quotes.fact}) AS {f_quotes.f_open_price};
+
+ALTER FACT {fact.quotes.open_price} ADD {f_quotes2.f_open_price};
+
+CREATE FOLDER {folder.quotes.attr}
+    VISUAL ( TITLE "Stock Quotes Data",
+             DESCRIPTION "Stock quotes data obtained from John Doe etc." )
+    TYPE ATTRIBUTE;
+
+ALTER DATATYPE {d_quotes_symbol.nm_symbol} VARCHAR(4),
+    {d_quotes_symbol.nm_symbol} VARCHAR(80),
+    {f_quotes.f_open_price} DECIMAL(10,2);
+
+INCLUDE TEMPLATE "URN:GOODDATA:DATE" MODIFY (IDENTIFIER "my-date", TITLE "quote")
+
+ALTER ATTRIBUTE {attr.quotes.symbol} ADD KEYS {d_quotes_symbol.nm_symbol} PRIMARY;
+
+ALTER ATTRIBUTE {attr.quotes.symbol} DROP KEYS {d_quotes_symbol.nm_symbol};
+
+ALTER FACT {fact.quotes.open_price} ADD {f_quotes2.f_open_price};
+
+# Another comment

File tests/examplefiles/test.scaml

+-@ import val city:String = "Tampa"
+- val name:String = "Hiram"
+%html
+  %body
+    %p Hello #{name} from #{city}
+    %ul
+    - for ( i <- 1 to 10 )
+      %li Item #{i}

File tests/examplefiles/test.ssp

+<%@ val someName: String = "someDefaultValue" %>
+<% import com.acme.MySnippets._ %>
+<html>
+<body>
+<h1>Hello ${someName}%</h1>
+
+<ul>
+<%= for (person <- people) { %>
+  <li>${person.name}</li>
+<% } %>
+</ul>  
+

File tests/test_basic_api.py

             inst = formatter()
             out = format(tokens, inst)
             assert type(out) is bytes, '%s: %r' % (formatter, out)
-    
+
     for formatter, info in formatters.FORMATTERS.iteritems():
         yield verify, formatter