Commits

Anonymous committed 68bfb51

#421: Emit Comment.Single or Comment.Multiline tokens for languages that have both variants.

Comments (0)

Files changed (5)

pygments/lexers/agile.py

             (r'\n', Text),
             (r'\s+', Text),
             # Comments
-            (r'//(.*?)\n', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
-            (r'/\+', Comment, 'nestedcomment'),
+            (r'//(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'/\+', Comment.Multiline, 'nestedcomment'),
             # Keywords
             (r'(as|assert|break|case|catch|class|continue|coroutine|default'
              r'|do|else|finally|for|foreach|function|global|namespace'
             (r'[a-zA-Z_]\w*', Name),
         ],
         'nestedcomment': [
-            (r'[^+/]+', Comment),
-            (r'/\+', Comment, '#push'),
-            (r'\+/', Comment, '#pop'),
-            (r'[+/]', Comment),
+            (r'[^+/]+', Comment.Multiline),
+            (r'/\+', Comment.Multiline, '#push'),
+            (r'\+/', Comment.Multiline, '#pop'),
+            (r'[+/]', Comment.Multiline),
         ],
     }
 
             (r'\n', Text),
             (r'\s+', Text),
             # Comments
-            (r'//(.*?)\n', Comment),
-            (r'#(.*?)\n', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
-            (r'/\+', Comment, 'nestedcomment'),
+            (r'//(.*?)\n', Comment.Single),
+            (r'#(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'/\+', Comment.Multiline, 'nestedcomment'),
             # DoubleQuotedString
             (r'"(\\\\|\\"|[^"])*"', String),
             # Operators
             (r'\d+', Number.Integer)
         ],
         'nestedcomment': [
-            (r'[^+/]+', Comment),
-            (r'/\+', Comment, '#push'),
-            (r'\+/', Comment, '#pop'),
-            (r'[+/]', Comment),
+            (r'[^+/]+', Comment.Multiline),
+            (r'/\+', Comment.Multiline, '#push'),
+            (r'\+/', Comment.Multiline, '#pop'),
+            (r'[+/]', Comment.Multiline),
         ]
     }
 

pygments/lexers/compiled.py

             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text), # line continuation
-            (r'//(\n|(.|\n)*?[^\\]\n)', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
+            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
         ],
         'statements': [
             (r'L?"', String, 'string'),
         ],
         'macro': [
             (r'[^/\n]+', Comment.Preproc),
-            (r'/[*](.|\n)*?[*]/', Comment),
-            (r'//.*?\n', Comment, '#pop'),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
             (r'/', Comment.Preproc),
             (r'(?<=\\)\n', Comment.Preproc),
             (r'\n', Comment.Preproc, '#pop'),
         ],
         'if0': [
-            (r'^\s*#if.*?(?<!\\)\n', Comment, '#push'),
+            (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
             (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
-            (r'^\s*#endif.*?(?<!\\)\n', Comment, '#pop'),
+            (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
             (r'.*?\n', Comment),
         ]
     }
             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text), # line continuation
-            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
+            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
             (r'[{}]', Punctuation),
             (r'L?"', String, 'string'),
             (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
         ],
         'macro': [
             (r'[^/\n]+', Comment.Preproc),
-            (r'/[*](.|\n)*?[*]/', Comment),
-            (r'//.*?\n', Comment, '#pop'),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
             (r'/', Comment.Preproc),
             (r'(?<=\\)\n', Comment.Preproc),
             (r'\n', Comment.Preproc, '#pop'),
         ],
         'if0': [
-            (r'^\s*#if.*?(?<!\\)\n', Comment, '#push'),
-            (r'^\s*#endif.*?(?<!\\)\n', Comment, '#pop'),
+            (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+            (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
             (r'.*?\n', Comment),
         ]
     }
             (r'\s+', Text),
             #(r'\\\n', Text), # line continuations
             # Comments
-            (r'//(.*?)\n', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
-            (r'/\+', Comment, 'nested_comment'),
+            (r'//(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'/\+', Comment.Multiline, 'nested_comment'),
             # Keywords
             (r'(abstract|alias|align|asm|assert|auto|body|break|case|cast'
              r'|catch|class|const|continue|debug|default|delegate|delete'
             (r'[a-zA-Z_]\w*', Name),
         ],
         'nested_comment': [
-            (r'[^+/]+', Comment),
-            (r'/\+', Comment, '#push'),
-            (r'\+/', Comment, '#pop'),
-            (r'[+/]', Comment),
+            (r'[^+/]+', Comment.Multiline),
+            (r'/\+', Comment.Multiline, '#push'),
+            (r'\+/', Comment.Multiline, '#pop'),
+            (r'[+/]', Comment.Multiline),
         ],
         'token_string': [
             (r'{', Punctuation, 'token_string_nest'),
              r'(\s*)(\()',                                  # signature start
              bygroups(using(this), Name.Function, Text, Operator)),
             (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment),
-            (r'/\*.*?\*/', Comment),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
             (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator),
             (r'(assert|break|case|catch|continue|default|do|else|finally|for|'
              r'if|goto|instanceof|new|return|switch|this|throw|try|while)\b',
             (r'(class|trait|object)(\s+)', bygroups(Keyword, Text), 'class'),
             (ur"'%s" % idrest, Text.Symbol),
             (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment),
+            (r'//.*?\n', Comment.Single),
             (r'/\*', Comment.Multiline, 'comment'),
             (ur'@%s' % idrest, Name.Decorator),
             (ur'(abstract|ca(?:se|tch)|d(?:ef|o)|e(?:lse|xtends)|'
              r'|v(ariable|irtual))\b', Name.Builtin),
             (r'<\w+>', Keyword.Type),
             (r'#?"(?:\\.|[^"])+?"', String.Double),
-            (r'//.*?\n', Comment),
+            (r'//.*?\n', Comment.Single),
             (r'/\*[\w\W]*?\*/', Comment.Multiline),
             (r'\'.*?\'', String.Single),
             (r'=>|\b(a(bove|fterwards)|b(e(gin|low)|y)|c(ase|leanup|reate)'
             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text), # line continuation
-            (r'//(\n|(.|\n)*?[^\\]\n)', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
+            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
         ],
         'statements': [
             (r'(L|@)?"', String, 'string'),
         ],
         'macro': [
             (r'[^/\n]+', Comment.Preproc),
-            (r'/[*](.|\n)*?[*]/', Comment),
-            (r'//.*?\n', Comment, '#pop'),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
             (r'/', Comment.Preproc),
             (r'(?<=\\)\n', Comment.Preproc),
             (r'\n', Comment.Preproc, '#pop'),
         ],
         'if0': [
-            (r'^\s*#if.*?(?<!\\)\n', Comment, '#push'),
-            (r'^\s*#endif.*?(?<!\\)\n', Comment, '#pop'),
+            (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
+            (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
             (r'.*?\n', Comment),
         ]
     }
 
     tokens = {
         'root': [
-            (r'^#.*', Comment),
-            (r'/\*', Comment, 'nested-comment'),
-            (r'%.*', Comment),
+            (r'^#.*', Comment.Single),
+            (r'/\*', Comment.Multiline, 'nested-comment'),
+            (r'%.*', Comment.Single),
             (r'[0-9]+', Number),
             (r'[\[\](){}|.,;!]', Punctuation),
             (r':-|-->', Punctuation),
             (u'\\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text),
         ],
         'nested-comment': [
-            (r'\*/', Comment, '#pop'),
-            (r'/\*', Comment, '#push'),
-            (r'[^*/]+', Comment),
-            (r'[*/]', Comment),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'[*/]', Comment.Multiline),
         ],
     }
 
             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text), # line continuation
-            (r'//(\n|(.|\n)*?[^\\]\n)', Comment),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment),
+            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
         ],
         'statements': [
             (r'L?"', String, 'string'),
             (r'\\', String), # stray backslash
         ],
         'if0': [
-            (r'^\s*#if.*?(?<!\\)\n', Comment, '#push'),
+            (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
             (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
-            (r'^\s*#endif.*?(?<!\\)\n', Comment, '#pop'),
+            (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
             (r'.*?\n', Comment),
         ],
         'class': [

pygments/lexers/dotnet.py

                 (r'^\s*\[.*?\]', Name.Attribute),
                 (r'[^\S\n]+', Text),
                 (r'\\\n', Text), # line continuation
-                (r'//.*?\n', Comment),
-                (r'/[*](.|\n)*?[*]/', Comment),
+                (r'//.*?\n', Comment.Single),
+                (r'/[*](.|\n)*?[*]/', Comment.Multiline),
                 (r'\n', Text),
                 (r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation),
                 (r'[{}]', Punctuation),
     tokens = {
         'root': [
             (r'\s+', Text),
-            (r'(#|//).*$', Comment),
-            (r'/[*]', Comment, 'comment'),
+            (r'(#|//).*$', Comment.Single),
+            (r'/[*]', Comment.Multiline, 'comment'),
             (r'[]{}:(),.;[]', Punctuation),
             (r'\\\n', Text),
             (r'\\', Text),

pygments/lexers/functional.py

                 line = match.group()
                 m = bird_re.match(line)
                 if m:
-                    insertions.append((len(code), [(0, Comment.Special, m.group(1))]))
+                    insertions.append((len(code),
+                                       [(0, Comment.Special, m.group(1))]))
                     code += m.group(2)
                 else:
                     insertions.append((len(code), [(0, Text, line)]))

pygments/lexers/web.py

         'commentsandwhitespace': [
             (r'\s+', Text),
             (r'<!--', Comment),
-            (r'//.*?\n', Comment),
-            (r'/\*.*?\*/', Comment)
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
         ],
         'slashstartsregex': [
             include('commentsandwhitespace'),
     tokens = {
         'root': [
             (r'\s+', Text),
-            (r'//.*?\n', Comment),
-            (r'/\*.*?\*/', Comment),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
             (r'/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex),
             (r'[~\^\*!%&<>\|+=:;,/?\\-]+', Operator),
             (r'[{}\[\]();.]+', Punctuation),
             (r'\?>', Comment.Preproc, '#pop'),
             (r'<<<([a-zA-Z_][a-zA-Z0-9_]*)\n.*?\n\1\;?\n', String),
             (r'\s+', Text),
-            (r'#.*?\n', Comment),
-            (r'//.*?\n', Comment),
-            (r'/\*\*/', Comment), # put the empty comment here, it is otherwise
-                                  # seen as the start of a docstring
+            (r'#.*?\n', Comment.Single),
+            (r'//.*?\n', Comment.Single),
+            # put the empty comment here, it is otherwise seen as
+            # the start of a docstring
+            (r'/\*\*/', Comment.Multiline),
             (r'/\*\*.*?\*/', String.Doc),
-            (r'/\*.*?\*/', Comment),
+            (r'/\*.*?\*/', Comment.Multiline),
             (r'(->|::)(\s*)([a-zA-Z_][a-zA-Z0-9_]*)',
              bygroups(Operator, Text, Name.Attribute)),
             (r'[~!%^&*+=|:.<>/?@-]+', Operator),