Anonymous avatar Anonymous committed 6835410

Fix Clojure's keyword/string/symbol lexing, and tidy up the code a bit as well

Comments (0)

Files changed (1)

pygments/lexers/agile.py

 
 from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, \
      LexerContext, include, combined, do_insertions, bygroups, using, this
-from pygments.token import Error, Text, Other, \
+from pygments.token import Error, Text, Whitespace, Other, \
      Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation
 from pygments.util import get_bool_opt, get_list_opt, shebang_matches
 from pygments import unistring as uni
     ]
     builtins = [
         '.', '..',
-        '*', '+', '-', '->', '..', '/', '<', '<=', '=', '==', '>', '>=',
+        '*', '+', '-', '->', '/', '<', '<=', '=', '==', '>', '>=',
         'accessor', 'agent', 'agent-errors', 'aget', 'alength', 'all-ns',
         'alter', 'and', 'append-child', 'apply', 'array-map', 'aset',
         'aset-boolean', 'aset-byte', 'aset-char', 'aset-double', 'aset-float',
     # valid names for identifiers
     # well, names can only not consist fully of numbers
     # but this should be good enough for now
-    valid_name = r'[\w!$%*+,<=>?@~-]+'
+
+    # TODO / should divide keywords/symbols into namespace/rest
+    # but that's hard, so just pretend / is part of the name
+    valid_name = r'[\w!$%*+,<=>?/-]+'
+
+    prefix_operators = ['`', "'", '#', '^', '~', '~@']
+
+    def _multi_escape(entries):
+        return '(%s)' % ('|'.join([
+                    re.escape(entry) + ' ' for entry in entries]))
 
     tokens = {
         'root' : [
             (r';.*$', Comment.Single),
 
             # whitespaces - usually not relevant
-            (r'\s+', Text),
+            (r'[,\s]+', Whitespace),
 
             # numbers
             (r'-?\d+\.\d+', Number.Float),
             (r'-?\d+', Number.Integer),
+            (r'0x-?[abcdef\d]+', Number.Hex),
 
             # strings, symbols and characters
             (r'"(\\\\|\\"|[^"])*"', String),
             (r"'" + valid_name, String.Symbol),
-            (r"\\([()/'\".'_!§$%& ?;=#+-]{1}|[a-zA-Z0-9]+)", String.Char),
+            (r"\\(.|[a-z]+)", String.Char),
 
-            # constants
+            # keywords
+            (r':' + valid_name, Name.Constant),
 
             # special operators
-            (r"('|#|`)", Operator),
+            (_multi_escape(prefix_operators),
+                Operator
+            ),
 
             # highlight the keywords
-            ('(%s)' % '|'.join([
-                re.escape(entry) + ' ' for entry in keywords]),
+            (_multi_escape(keywords),
                 Keyword
             ),
 
             # highlight the builtins
-            ("(?<=\()(%s)" % '|'.join([
-                re.escape(entry) + ' ' for entry in builtins]),
+            (_multi_escape(builtins),
                 Name.Builtin
             ),
 
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.