1. Oldes
  2. pygments-main

Commits

Oldes  committed bcb1a2c

Red language (red-lang.org) lexer - initial version

  • Participants
  • Parent commits 6244c94
  • Branches Red-language-Lexer

Comments (0)

Files changed (4)

File pygments/lexers/_mapping.py

View file
  • Ignore whitespace
     'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)),
     'RdLexer': ('pygments.lexers.math', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)),
     'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3'), ('text/x-rebol',)),
+	'RedLexer': ('pygments.lexers.other', 'Red', ('red','red/system'), ('*.red', '*.reds'), ('text/x-red','text/x-red-system',)),
     'RedcodeLexer': ('pygments.lexers.other', 'Redcode', ('redcode',), ('*.cw',), ()),
     'RegeditLexer': ('pygments.lexers.text', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)),
     'RexxLexer': ('pygments.lexers.other', 'Rexx', ('rexx', 'ARexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),

File pygments/lexers/other.py

View file
  • Ignore whitespace
            'ECLLexer', 'UrbiscriptLexer', 'OpenEdgeLexer', 'BroLexer',
            'MscgenLexer', 'KconfigLexer', 'VGLLexer', 'SourcePawnLexer',
            'RobotFrameworkLexer', 'PuppetLexer', 'NSISLexer', 'RPMSpecLexer',
-           'CbmBasicV2Lexer', 'AutoItLexer', 'RexxLexer']
+           'CbmBasicV2Lexer', 'AutoItLexer', 'RexxLexer', 'RedLexer']
 
 
 class ECLLexer(RegexLexer):
         ]
     }
 
+
+class RedLexer(RegexLexer):
+    """
+    A `Red-language <http://www.red-lang.org/>`_ lexer.
+
+    *New in Pygments 1.6.*
+    """
+    name = 'Red'
+    aliases = ['red', 'red/system']
+    filenames = ['*.red', '*.reds']
+    mimetypes = ['text/x-red', 'text/x-red-system']
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    re.IGNORECASE
+
+    escape_re = r'(?:\^\([0-9a-fA-F]{1,4}\)*)'
+
+    def word_callback(lexer, match):
+        word = match.group()
+
+        if re.match(".*:$", word):
+            yield match.start(), Generic.Subheading, word
+        elif re.match(
+            r'(if|unless|either|any|all|while|until|loop|repeat|'
+            r'foreach|forall|func|function|does|has|switch|'
+            r'case|reduce|compose|get|set|print|prin|equal\?|'
+            r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|'
+            r'greater-or-equal\?|same\?|not|type\?|stats|'
+            r'bind|union|replace|charset)$', word):
+            yield match.start(), Name.Builtin, word
+        elif re.match(
+            r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|'
+            r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|'
+            r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|'
+            r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|'
+            r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|update|write)$', word):
+            yield match.start(), Name.Function, word
+        elif re.match(
+            r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|none|crlf|dot|null-byte)$', word):
+            yield match.start(), Name.Builtin.Pseudo, word
+        elif re.match(
+            r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|#switch|#default|#get-definition)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match(
+            r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|raise-error|'
+            r'return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|quote|forever)$', word):
+            yield match.start(), Name.Exception, word
+        elif re.match(
+            r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|any-struct\?|'
+            r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|none\?|'
+            r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|word\?|any-series\?)$', word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match("to-.*", word):
+            yield match.start(), Keyword, word
+        elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|===|<>|<|>|<=|>=|<<|>>|<<<|>>>|%|-\*\*)$', word):
+            yield match.start(), Operator, word
+        elif re.match(".*\!$", word):
+            yield match.start(), Keyword.Type, word
+        elif re.match("'.*", word):
+            yield match.start(), Name.Variable.Instance, word # lit-word
+        elif re.match("#.*", word):
+            yield match.start(), Name.Label, word # issue
+        elif re.match("%.*", word):
+            yield match.start(), Name.Decorator, word # file
+        elif re.match(":.*", word):
+            yield match.start(), Generic.Subheading, word # get-word
+        else:
+            yield match.start(), Name.Variable, word
+
+    tokens = {
+        'root': [
+            (r'[^R]+', Comment),
+            (r'Red/System\s+\[', Generic.Strong, 'script'),
+            (r'Red\s+\[', Generic.Strong, 'script'),
+            (r'R', Comment)
+        ],
+        'script': [
+            (r'\s+', Text),
+            (r'#"', String.Char, 'char'),
+            (r'#{[0-9a-fA-F]*}', Number.Hex),
+            (r'2#{', Number.Hex, 'bin2'),
+            (r'64#{[0-9a-zA-Z+/=\s]*}', Number.Hex),
+			(r'([0-9a-fA-F]+)(h)',  bygroups(Number.Hex, Name.Variable)),
+            (r'"', String, 'string'),
+            (r'{', String, 'string2'),
+            (r';#+.*\n', Comment.Special),
+            (r';\*+.*\n', Comment.Preproc),
+            (r';.*\n', Comment),
+            (r'%"', Name.Decorator, 'stringFile'),
+            (r'%[^(\^{^")\s\[\]]+', Name.Decorator),
+            (r'[+-]?([a-zA-Z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money
+            (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time
+            (r'\d+[\-\/][0-9a-zA-Z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?'
+             r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date
+            (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple
+            (r'\d+[xX]\d+', Keyword.Constant), # pair
+            (r'[+-]?\d+(\'\d+)?([\.,]\d*)?[eE][+-]?\d+', Number.Float),
+            (r'[+-]?\d+(\'\d+)?[\.,]\d*', Number.Float),
+            (r'[+-]?\d+(\'\d+)?', Number),
+            (r'[\[\]\(\)]', Generic.Strong),
+            (r'[a-zA-Z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url
+            (r'mailto:[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # url
+            (r'[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # email
+            (r'comment\s', Comment, 'comment'),
+            (r'/[^(\^{^")\s/[\]]*', Name.Attribute),
+            (r'([^(\^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+            (r'<[a-zA-Z0-9:._-]*>', Name.Tag),
+            (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+            (r'([^(\^{^")\s]+)', Text),
+        ],
+        'string': [
+            (r'[^(\^")]+', String),
+            (escape_re, String.Escape),
+            (r'[\(|\)]+', String),
+            (r'\^.', String.Escape),
+            (r'"', String, '#pop'),
+        ],
+        'string2': [
+            (r'[^(\^{^})]+', String),
+            (escape_re, String.Escape),
+            (r'[\(|\)]+', String),
+            (r'\^.', String.Escape),
+            (r'{', String, '#push'),
+            (r'}', String, '#pop'),
+        ],
+        'stringFile': [
+            (r'[^(\^")]+', Name.Decorator),
+            (escape_re, Name.Decorator),
+            (r'\^.', Name.Decorator),
+            (r'"', Name.Decorator, '#pop'),
+        ],
+        'char': [
+            (escape_re + '"', String.Char, '#pop'),
+            (r'\^."', String.Char, '#pop'),
+            (r'."', String.Char, '#pop'),
+        ],
+        'tag': [
+            (escape_re, Name.Tag),
+            (r'"', Name.Tag, 'tagString'),
+            (r'[^(<>\r\n")]+', Name.Tag),
+            (r'>', Name.Tag, '#pop'),
+        ],
+        'tagString': [
+            (r'[^(\^")]+', Name.Tag),
+            (escape_re, Name.Tag),
+            (r'[\(|\)]+', Name.Tag),
+            (r'\^.', Name.Tag),
+            (r'"', Name.Tag, '#pop'),
+        ],
+        'tuple': [
+            (r'(\d+\.)+', Keyword.Constant),
+            (r'\d+', Keyword.Constant, '#pop'),
+        ],
+        'bin2': [
+            (r'\s+', Number.Hex),
+            (r'([0-1]\s*){8}', Number.Hex),
+            (r'}', Number.Hex, '#pop'),
+        ],
+        'comment': [
+            (r'"', Comment, 'commentString1'),
+            (r'{', Comment, 'commentString2'),
+            (r'\[', Comment, 'commentBlock'),
+            (r'[^(\s{\"\[]+', Comment, '#pop'),
+        ],
+        'commentString1': [
+            (r'[^(\^")]+', Comment),
+            (escape_re, Comment),
+            (r'[\(|\)]+', Comment),
+            (r'\^.', Comment),
+            (r'"', Comment, '#pop'),
+        ],
+        'commentString2': [
+            (r'[^(\^{^})]+', Comment),
+            (escape_re, Comment),
+            (r'[\(|\)]+', Comment),
+            (r'\^.', Comment),
+            (r'{', Comment, '#push'),
+            (r'}', Comment, '#pop'),
+        ],
+        'commentBlock': [
+            (r'\[', Comment, '#push'),
+            (r'\]', Comment, '#pop'),
+            (r'[^(\[\])]+', Comment),
+        ],
+    }
+
     _c = lambda s: re.compile(s, re.MULTILINE)
     _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
     _ADDRESS_PATTERN = _c(r'^\s*address\s+')

File tests/examplefiles/example.red

View file
  • Ignore whitespace
+Red [
+    Title:  "Red console"
+    Author: ["Nenad Rakocevic" "Kaj de Vos"]
+    File:   %console.red
+    Tabs:   4
+    Rights: "Copyright (C) 2012-2013 Nenad Rakocevic. All rights reserved."
+    License: {
+        Distributed under the Boost Software License, Version 1.0.
+        See https://github.com/dockimbel/Red/blob/master/BSL-License.txt
+    }
+    Purpose:    "Just some code for testing Pygments colorizer"
+    Language:   http://www.red-lang.org/
+]
+
+#system-global [
+    #either OS = 'Windows [
+        #import [
+            "kernel32.dll" stdcall [
+                AttachConsole:   "AttachConsole" [
+                    processID       [integer!]
+                    return:         [integer!]
+                ]
+                SetConsoleTitle: "SetConsoleTitleA" [
+                    title           [c-string!]
+                    return:         [integer!]
+                ]
+                ReadConsole:     "ReadConsoleA" [
+                    consoleInput    [integer!]
+                    buffer          [byte-ptr!]
+                    charsToRead     [integer!]
+                    numberOfChars   [int-ptr!]
+                    inputControl    [int-ptr!]
+                    return:         [integer!]
+                ]
+            ]
+        ]
+        line-buffer-size: 16 * 1024
+        line-buffer: allocate line-buffer-size
+    ][
+        #switch OS [
+            MacOSX [
+                #define ReadLine-library "libreadline.dylib"
+            ]
+            #default [
+                #define ReadLine-library "libreadline.so.6"
+                #define History-library  "libhistory.so.6"
+            ]
+        ]
+        #import [
+            ReadLine-library cdecl [
+                read-line: "readline" [  ; Read a line from the console.
+                    prompt          [c-string!]
+                    return:         [c-string!]
+                ]
+                rl-bind-key: "rl_bind_key" [
+                    key             [integer!]
+                    command         [integer!]
+                    return:         [integer!]
+                ]
+                rl-insert:   "rl_insert" [
+                    count           [integer!]
+                    key             [integer!]
+                    return:         [integer!]
+                ]
+            ]
+            #if OS <> 'MacOSX [
+                History-library cdecl [
+                    add-history: "add_history" [  ; Add line to the history.
+                        line        [c-string!]
+                    ]
+                ]
+            ]
+        ]
+
+        rl-insert-wrapper: func [
+            [cdecl]
+            count   [integer!]
+            key     [integer!]
+            return: [integer!]
+        ][
+            rl-insert count key
+        ]
+        
+    ]
+]
+
+Windows?: system/platform = 'Windows
+
+read-argument: routine [
+    /local
+        args [str-array!]
+        str  [red-string!]
+][
+    if system/args-count <> 2 [
+        SET_RETURN(none-value)
+        exit
+    ]
+    args: system/args-list + 1                          ;-- skip binary filename
+    str: simple-io/read-txt args/item
+    SET_RETURN(str)
+]
+
+init-console: routine [
+    str [string!]
+    /local
+        ret
+][
+    #either OS = 'Windows [
+        ;ret: AttachConsole -1
+        ;if zero? ret [print-line "ReadConsole failed!" halt]
+        
+        ret: SetConsoleTitle as c-string! string/rs-head str
+        if zero? ret [print-line "SetConsoleTitle failed!" halt]
+    ][
+        rl-bind-key as-integer tab as-integer :rl-insert-wrapper
+    ]
+]
+
+input: routine [
+    prompt [string!]
+    /local
+        len ret str buffer line
+][
+    #either OS = 'Windows [
+        len: 0
+        print as c-string! string/rs-head prompt
+        ret: ReadConsole stdin line-buffer line-buffer-size :len null
+        if zero? ret [print-line "ReadConsole failed!" halt]
+        len: len + 1
+        line-buffer/len: null-byte
+        str: string/load as c-string! line-buffer len
+    ][
+        line: read-line as c-string! string/rs-head prompt
+        if line = null [halt]  ; EOF
+
+         #if OS <> 'MacOSX [add-history line]
+
+        str: string/load line  1 + length? line
+;       free as byte-ptr! line
+    ]
+    SET_RETURN(str)
+]
+
+count-delimiters: function [
+    buffer  [string!]
+    return: [block!]
+][
+    list: copy [0 0]
+    c: none
+    
+    foreach c buffer [
+        case [
+            escaped? [
+                escaped?: no
+            ]
+            in-comment? [
+                switch c [
+                    #"^/" [in-comment?: no]
+                ]
+            ]
+            'else [
+                switch c [
+                    #"^^" [escaped?: yes]
+                    #";"  [if zero? list/2 [in-comment?: yes]]
+                    #"["  [list/1: list/1 + 1]
+                    #"]"  [list/1: list/1 - 1]
+                    #"{"  [list/2: list/2 + 1]
+                    #"}"  [list/2: list/2 - 1]
+                ]
+            ]
+        ]
+    ]
+    list
+]
+
+do-console: function [][
+    buffer: make string! 10000
+    prompt: red-prompt: "red>> "
+    mode:  'mono
+    
+    switch-mode: [
+        mode: case [
+            cnt/1 > 0 ['block]
+            cnt/2 > 0 ['string]
+            'else     [
+                prompt: red-prompt
+                do eval
+                'mono
+            ]
+        ]
+        prompt: switch mode [
+            block  ["[^-"]
+            string ["{^-"]
+            mono   [red-prompt]
+        ]
+    ]
+    
+    eval: [
+        code: load/all buffer
+        
+        unless tail? code [
+            set/any 'result do code
+            
+            unless unset? :result [
+                if 67 = length? result: mold/part :result 67 [  ;-- optimized for width = 72
+                    clear back tail result
+                    append result "..."
+                ]
+                print ["==" result]
+            ]
+        ]
+        clear buffer
+    ]
+
+    while [true][
+        unless tail? line: input prompt [
+            append buffer line
+            cnt: count-delimiters buffer
+
+            either Windows? [
+                remove skip tail buffer -2          ;-- clear extra CR (Windows)
+            ][
+                append buffer lf                    ;-- Unix
+            ]
+            
+            switch mode [
+                block  [if cnt/1 <= 0 [do switch-mode]]
+                string [if cnt/2 <= 0 [do switch-mode]]
+                mono   [do either any [cnt/1 > 0 cnt/2 > 0][switch-mode][eval]]
+            ]
+        ]
+    ]
+]
+
+q: :quit
+
+if script: read-argument [
+    script: load script
+    either any [
+        script/1 <> 'Red
+        not block? script/2 
+    ][
+        print "*** Error: not a Red program!"
+    ][
+        do skip script 2
+    ]
+    quit
+]
+
+init-console "Red Console"
+
+print {
+-=== Red Console alpha version ===-
+(only ASCII input supported)
+}
+
+do-console

File tests/examplefiles/example.reds

View file
  • Ignore whitespace
+Red/System [
+    Title:      "Red/System example file"
+    Purpose:    "Just some code for testing Pygments colorizer"
+    Language:   http://www.red-lang.org/
+]
+
+#include %../common/FPU-configuration.reds
+
+; C types
+
+#define time!                   long!
+#define clock!                  long!
+
+date!: alias struct! [
+    second                      [integer!]  ; 0-61 (60?)
+    minute                      [integer!]  ; 0-59
+    hour                        [integer!]  ; 0-23
+
+    day                         [integer!]  ; 1-31
+    month                       [integer!]  ; 0-11
+    year                        [integer!]  ; Since 1900
+
+    weekday                     [integer!]  ; 0-6 since Sunday
+    yearday                     [integer!]  ; 0-365
+    daylight-saving-time?       [integer!]  ; Negative: unknown
+]
+
+#either OS = 'Windows [
+    #define clocks-per-second   1000
+][
+    ; CLOCKS_PER_SEC value for Syllable, Linux (XSI-conformant systems)
+    ; TODO: check for other systems
+    #define clocks-per-second   1000'000
+] 
+
+#import [LIBC-file cdecl [
+
+    ; Error handling
+
+    form-error: "strerror" [  ; Return error description.
+        code            [integer!]
+        return:         [c-string!]
+    ]
+    print-error: "perror" [  ; Print error to standard error output.
+        string          [c-string!]
+    ]
+
+
+    ; Memory management
+
+    make: "calloc" [  ; Allocate zero-filled memory.
+        chunks          [size!]
+        size            [size!]
+        return:         [binary!]
+    ]
+    resize: "realloc" [  ; Resize memory allocation.
+        memory          [binary!]
+        size            [size!]
+        return:         [binary!]
+    ]
+ ]
+ 
+ JVM!: alias struct! [
+    reserved0                   [int-ptr!]
+    reserved1                   [int-ptr!]
+    reserved2                   [int-ptr!]
+    
+    DestroyJavaVM               [function! [[JNICALL] vm [JVM-ptr!] return: [jint!]]]
+    AttachCurrentThread         [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] args [byte-ptr!] return: [jint!]]]
+    DetachCurrentThread         [function! [[JNICALL] vm [JVM-ptr!] return: [jint!]]]
+    GetEnv                      [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] version [integer!] return: [jint!]]]
+    AttachCurrentThreadAsDaemon [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] args [byte-ptr!] return: [jint!]]]
+]
+
+ ;just some datatypes for testing:
+ 
+ #some-hash
+ 10-1-2013
+ quit
+ 
+ ;some numbers:
+ 12
+ 1'000
+ 1.2
+ FF00FF00h
+
+;normal words:
+foo
+
+;get-word
+:foo
+ 
+;lit-word:
+'foo 'foo
+
+to-integer foo
+foo/(a + 1)/b
+
+call/output reform ['which interpreter] path: copy ""
+
+ version-1.1:   00010001h
+ 
+ #if type = 'exe [
+    push system/stack/frame                 ;-- save previous frame pointer
+    system/stack/frame: system/stack/top    ;-- @@ reposition frame pointer just after the catch flag
+]
+push CATCH_ALL                              ;-- exceptions root barrier
+push 0                                      ;-- keep stack aligned on 64-bit