Commits

David Corbett committed 1c4b3df

Add token type Number.Bin

Comments (0)

Files changed (12)

doc/docs/tokens.rst

 `Number`
     Token type for any number literal.
 
+`Number.Bin`
+    Token type for binary literals (e.g. ``0b101010``).
+
 `Number.Float`
     Token type for float literals (e.g. ``42.0``).
 

pygments/lexers/agile.py

              Number.Float),
             # IntegerLiteral
             # -- Binary
-            (r'0[bB][01][01_]*', Number),
+            (r'0[bB][01][01_]*', Number.Bin),
             # -- Hexadecimal
             (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex),
             # -- Decimal
             (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number),
             (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
             (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
-            (r'0b[01]+\s', Number),
-            (r'0o[0-7]+\s', Number),
+            (r'0b[01]+\s', Number.Bin),
+            (r'0o[0-7]+\s', Number.Oct),
             (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
             (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
 

pygments/lexers/asm.py

             (string, String),
             (hexn, Number.Hex),
             (octn, Number.Oct),
-            (binn, Number),
+            (binn, Number.Bin),
             (floatn, Number.Float),
             (decn, Number.Integer),
             include('punctuation'),
             (r'"[^"\n]*.', String),
             (r"'[^'\n]*.", String.Char),
             (r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
-            (r'\d+|%[01]+', Number.Integer),
+            (r'\d+', Number.Integer),
+            (r'%[01]+', Number.Bin),
             (r'[#,.:()=]', Punctuation),
             (r'[a-z_.@$][\w.@$]*', Name),
         ]

pygments/lexers/compiled.py

             (r'\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[fFL]?[i]?', Number.Float),
             # IntegerLiteral
             # -- Binary
-            (r'0[Bb][01_]+', Number),
+            (r'0[Bb][01_]+', Number.Bin),
             # -- Octal
             (r'0[0-7_]+', Number.Oct),
             # -- Hexadecimal
             (r"'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\\'\n])'", String.Char),
 
             # binary integer
-            (r'#[bB][01]+', Number),
+            (r'#[bB][01]+', Number.Bin),
 
             # octal integer
             (r'#[oO][0-7]+', Number.Oct),
 
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'0c[0-9]+', Number.Oct),
-            (r'0b[01]+', Number.Binary),
+            (r'0b[01]+', Number.Bin),
             (r'[0-9_]\.[0-9_]*(?!\.)', Number.Float),
             (r'[0-9_]+', Number.Decimal),
 
 
             # IntegerLiteral
             # -- Binary
-            (r'0[Bb][01_]+%s' % decimal_suffixes, Number),
+            (r'0[Bb][01_]+%s' % decimal_suffixes, Number.Bin),
             # -- Octal
             (r'0[0-7_]+%s' % decimal_suffixes, Number.Oct),
             # -- Hexadecimal
             (r'([a-zA-Z_\$][\w\$]*)', Name),
         ],
         'numliterals': [
-            (r'[01]+B', Number.Binary),        # binary number (ObjM2)
+            (r'[01]+B', Number.Bin),           # binary number (ObjM2)
             (r'[0-7]+B', Number.Oct),          # octal number (PIM + ISO)
             (r'[0-7]+C', Number.Oct),          # char code (PIM + ISO)
             (r'[0-9A-F]+C', Number.Hex),       # char code (ObjM2)
             (r'\.[0-9]*(?!\.)', Number.Float),
             (r'[0-9]+', Number.Integer),
             (r'\$[0-9a-f]+', Number.Hex),
-            (r'\%[10]+', Number), # Binary
+            (r'\%[10]+', Number.Bin),
             # Other
             (r'(?:(?:(:)?([ \t]*)(:?%s|([+\-*/&|~]))|Or|And|Not|[=<>^]))' %
              (bmax_vopwords), Operator),
             (r'\.[0-9]+(?!\.)', Number.Float),
             (r'[0-9]+', Number.Integer),
             (r'\$[0-9a-f]+', Number.Hex),
-            (r'\%[10]+', Number), # Binary
+            (r'\%[10]+', Number.Bin),
             # Other
             (r'(?:%s|([+\-*/~=<>^]))' % (bb_vopwords), Operator),
             (r'[(),:\[\]\\]', Punctuation),
             (r'[0-9][0-9_]*(?=([eE.]|\'[fF](32|64)))',
               Number.Float, ('float-suffix', 'float-number')),
             (r'0[xX][a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'),
-            (r'0[bB][01][01_]*', Number, 'int-suffix'),
+            (r'0[bB][01][01_]*', Number.Bin, 'int-suffix'),
             (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'),
             (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'),
             # Whitespace
             # Lifetime
             (r"""'[a-zA-Z_]\w*""", Name.Label),
             # Binary Literal
-            (r'0b[01_]+', Number, 'number_lit'),
+            (r'0b[01_]+', Number.Bin, 'number_lit'),
             # Octal Literal
             (r'0o[0-7_]+', Number.Oct, 'number_lit'),
             # Hexadecimal Literal
             (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
             # Decimal Literal
-            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?'
-             r'[0-9_]+|\.[0-9_]*|[eE][+\-]?[0-9_]+)?', Number, 'number_lit'),
+            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+             r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'),
+            (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
             # String Literal
             (r'"', String, 'string'),
             (r'r(#*)".*?"\1', String.Raw),
             (r'\.[0-9]+(?!\.)', Number.Float),
             (r'[0-9]+', Number.Integer),
             (r'\$[0-9a-fA-Z]+', Number.Hex),
-            (r'\%[10]+', Number), # Binary
+            (r'\%[10]+', Number.Bin),
             # Native data types
             (r'\b%s\b' % keyword_type, Keyword.Type),
             # Exception handling
 
             # integer literals
             # -- binary
-            (r'0[bB][0-1]+', Number),
+            (r'0[bB][0-1]+', Number.Bin),
             # -- hex
             (r'0[xX][0-9a-fA-F]+', Number.Hex),
             # -- decimal
             (r'\$[+%s][0-9]*\.?[0-9]*([eE][+%s]?[0-9]+)?' % (_dash, _dash),
              Number.Float, '#pop'),
             (r'\$[0-9a-fA-F]+', Number.Hex, '#pop'),
-            (r'\$\$[01]+', Number, '#pop'),  # Binary
+            (r'\$\$[01]+', Number.Bin, '#pop'),
             (r'[0-9]+', Number.Integer, '#pop'),
             # Values prefixed by hashes
             (r'(##|#a\$)(%s)' % _name, bygroups(Operator, Name), '#pop'),

pygments/lexers/dotnet.py

             (r'\d[\d_]*[uU]?[yslLnQRZINGmM]?', Number.Integer),
             (r'0[xX][\da-fA-F][\da-fA-F_]*[uU]?[yslLn]?[fF]?', Number.Hex),
             (r'0[oO][0-7][0-7_]*[uU]?[yslLn]?', Number.Oct),
-            (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Binary),
+            (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Bin),
             (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)[fFmM]?',
              Number.Float),
 

pygments/lexers/functional.py

     _closing_parenthesis = r'[)\]}]'
     _delimiters = r'()[\]{}",\'`;\s'
     _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters
-    _number_prefix = r'(?:#e)?(?:#b|(?:#d)?)(?:#e)?'
+    _exact_decimal_prefix = r'(?:#e)?(?:#d)?(?:#e)?'
     _exponent = r'(?:[defls][-+]?\d+)'
     _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)'
     _inexact_simple = (r'(?:%s|(?:\d+#+(?:\.#*|/\d+#*)?|\.\d+#+|'
             # can denote the base or the type. These don't map neatly
             # onto Pygments token types; some judgment calls here.
 
-            # #b or #d or no prefix
-            (r'(?i)%s[-+]?\d+(?=[%s])' % (_number_prefix, _delimiters),
+            # #d or no prefix
+            (r'(?i)%s[-+]?\d+(?=[%s])' % (_exact_decimal_prefix, _delimiters),
              Number.Integer, '#pop'),
             (r'(?i)%s[-+]?(\d+(\.\d*)?|\.\d+)([deflst][-+]?\d+)?(?=[%s])' %
-             (_number_prefix, _delimiters), Number.Float, '#pop'),
+             (_exact_decimal_prefix, _delimiters), Number.Float, '#pop'),
             (r'(?i)%s[-+]?(%s([-+]%s?i)?|[-+]%s?i)(?=[%s])' %
-             (_number_prefix, _inexact_normal_no_hashes,
+             (_exact_decimal_prefix, _inexact_normal_no_hashes,
               _inexact_normal_no_hashes, _inexact_normal_no_hashes,
               _delimiters), Number, '#pop'),
 
             # Inexact without explicit #i
-            (r'(?i)(#[bd])?(%s([-+]%s?i)?|[-+]%s?i|%s@%s)(?=[%s])' %
+            (r'(?i)(#d)?(%s([-+]%s?i)?|[-+]%s?i|%s@%s)(?=[%s])' %
              (_inexact_real, _inexact_unsigned, _inexact_unsigned,
               _inexact_real, _inexact_real, _delimiters), Number.Float,
              '#pop'),
             (r'(?i)(([-+]?%st[-+]?\d+)|[-+](inf|nan)\.t)(?=[%s])' %
              (_inexact_simple, _delimiters), Number.Float, '#pop'),
 
+            # #b
+            (r'(?i)(#[ei])?#b%s' % _symbol, Number.Bin, '#pop'),
+
             # #o
             (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'),
 
             (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'),
 
             # #i is always inexact, i.e. float
-            (r'(?i)(#[bd])?#i%s' % _symbol, Number.Float, '#pop'),
+            (r'(?i)(#d)?#i%s' % _symbol, Number.Float, '#pop'),
 
             # Strings and characters
             (r'#?"', String.Double, ('#pop', 'string')),
             (r'#\'', Name.Function),
 
             # binary rational
-            (r'#[bB][+-]?[01]+(/[01]+)?', Number),
+            (r'#[bB][+-]?[01]+(/[01]+)?', Number.Bin),
 
             # octal rational
             (r'#[oO][+-]?[0-7]+(/[0-7]+)?', Number.Oct),
             (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
             (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
             (r'0[oO][0-7][0-7_]*', Number.Oct),
-            (r'0[bB][01][01_]*', Number.Binary),
+            (r'0[bB][01][01_]*', Number.Bin),
             (r'\d[\d_]*', Number.Integer),
 
             (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
             (r'-?\d+[eE][+\-]?\d+', Number.Float),
             (r'0[xX][\da-fA-F]+', Number.Hex),
             (r'0[oO][0-7]+', Number.Oct),
-            (r'0[bB][01]+', Number.Binary),
+            (r'0[bB][01]+', Number.Bin),
             (r'\d+', Number.Integer),
             # color literals
             (r'#[\da-fA-F]{3,6}', Number.Integer),
             (r'\d[\d_]*', Number.Integer),
             (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
             (r'0[oO][0-7][0-7_]*', Number.Oct),
-            (r'0[bB][01][01_]*', Number.Binary),
+            (r'0[bB][01][01_]*', Number.Bin),
             (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
 
             (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
             (r'[a-zA-Z_!]\w*[!\?]?', Name),
             (r'[(){};,/\|:\\\[\]]', Punctuation),
             (r'@[a-zA-Z_]\w*|&\d', Name.Variable),
-            (r'\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])'
-             r'(_?\d)*)?([eE][-+]?\d(_?\d)*)?|0[bB][01]+)\b', Number),
+            (r'(?i)\b(0x[\da-f]+)\b', Number.Hex),
+            (r'\b\d(_?\d)*(?!\.)\b', Number.Integer),
+            (r'\b(\d(_?\d)*(\.(?![^\d\s])(_?\d)*)?([eE][-+]?\d(_?\d)*)?)\b',
+             Number.Float),
+            (r'\b(0[bB][01]+)\b', Number.Bin),
             (r'%r\/.*\/', String.Regex),
             include('strings'),
         ],

pygments/lexers/hdl.py

             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
             (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
             (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
-            (r'([0-9]+)|(\'b)[0-1]+', Number.Hex),   # should be binary
+            (r'([0-9]+)|(\'b)[0-1]+', Number.Bin),
             (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
             (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
             (r'\'[01xz]', Number),
             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
             (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
             (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
-            (r'([0-9]+)|(\'b)[0-1]+', Number.Hex),   # should be binary
+            (r'([0-9]+)|(\'b)[0-1]+', Number.Bin),
             (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
             (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
             (r'\'[01xz]', Number),
         ],
         'numbers': [
             (r'\d{1,2}#[0-9a-f_]+#?', Number.Integer),
-            (r'[0-1_]+(\.[0-1_])', Number.Integer),
             (r'\d+', Number.Integer),
             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
-            (r'H"[0-9a-f_]+"', Number.Oct),
+            (r'X"[0-9a-f_]+"', Number.Hex),
             (r'O"[0-7_]+"', Number.Oct),
-            (r'B"[0-1_]+"', Number.Oct),
+            (r'B"[0-1_]+"', Number.Bin),
         ],
     }

pygments/lexers/jvm.py

              Number.Float),
             (r'#([0-9a-fA-F]{4})(_[0-9a-fA-F]{4})+', Number.Hex),
             (r'#[0-9a-fA-F]+', Number.Hex),
-            (r'\$([01]{4})(_[01]{4})+', Number.Integer),
-            (r'\$[01]+', Number.Integer),
+            (r'\$([01]{4})(_[01]{4})+', Number.Bin),
+            (r'\$[01]+', Number.Bin),
             (r'\d{1,3}(_\d{3})+[kMGTP]?', Number.Integer),
             (r'[0-9]+[kMGTP]?', Number.Integer),
             (r'\n', Text)

pygments/lexers/math.py

             (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
             (r'\d+(_\d+)+[eEf][+-]?[0-9]+', Number.Float),
             (r'\d+[eEf][+-]?[0-9]+', Number.Float),
-            (r'0b[01]+(_[01]+)+', Number.Binary),
-            (r'0b[01]+', Number.Binary),
+            (r'0b[01]+(_[01]+)+', Number.Bin),
+            (r'0b[01]+', Number.Bin),
             (r'0o[0-7]+(_[0-7]+)+', Number.Oct),
             (r'0o[0-7]+', Number.Oct),
             (r'0x[a-fA-F0-9]+(_[a-fA-F0-9]+)+', Number.Hex),

pygments/lexers/other.py

             (r'\s+', Text),
             # Numbers
             (r"0'.", Number),
-            (r'0b[01]+', Number),
-            (r'0o[0-7]+', Number),
-            (r'0x[0-9a-fA-F]+', Number),
-            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number.Float),
             # Variables
             (r'([A-Z_]\w*)', Name.Variable),
             # Event handlers
              r'(?=[(])', Keyword),
             # Numbers
             (r"0'.", Number),
-            (r'0b[01]+', Number),
-            (r'0o[0-7]+', Number),
-            (r'0x[0-9a-fA-F]+', Number),
-            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number.Float),
             # Variables
             (r'([A-Z_]\w*)', Name.Variable),
             # Atoms

pygments/lexers/web.py

             (r'0[0-7]+', Number.Oct),
             (r'0[xX][a-f0-9]+', Number.Hex),
             (r'\d+', Number.Integer),
-            (r'0b[01]+', Number.Binary),
+            (r'0b[01]+', Number.Bin),
             (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
             (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
             (r'"', String.Double, 'string'),
 
             # handle operator state
             # order on numbers matters - handle most complex first
-            (r'\d+(\.\d*)?[eE][\+\-]?\d+', Number.Double, 'operator'),
-            (r'(\.\d+)[eE][\+\-]?\d+', Number.Double, 'operator'),
-            (r'(\.\d+|\d+\.\d*)', Number, 'operator'),
+            (r'\d+(\.\d*)?[eE][\+\-]?\d+', Number.Float, 'operator'),
+            (r'(\.\d+)[eE][\+\-]?\d+', Number.Float, 'operator'),
+            (r'(\.\d+|\d+\.\d*)', Number.Float, 'operator'),
             (r'(\d+)', Number.Integer, 'operator'),
             (r'(\.\.|\.|\))', Punctuation, 'operator'),
             (r'(declare)(\s+)(construction)',

pygments/token.py

     String.Symbol:                 'ss',
 
     Number:                        'm',
+    Number.Bin:                    'mb',
     Number.Float:                  'mf',
     Number.Hex:                    'mh',
     Number.Integer:                'mi',