Commits

Ned Batchelder committed 5b650d4

Accept octal integer literals.

Comments (0)

Files changed (2)

 import re
 
 class Tok(object):
+    """A specification for a token class."""
+
     num = 0
 
     def __init__(self, name, regex, next=None):
     def lex(self, text):
         """Lexically analyze `text`.
 
-        Yields pairs (`name`, `token`).
+        Yields pairs (`name`, `tokentext`).
 
         """
         while text:
         Tok("reserved",     literals("null true false"), next='div'),
         Tok("id",           r"([a-zA-Z_$]|\\u[0-9a-fA-Z]{4})([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*", next='div'),
         Tok("hnum",         r"0[xX][0-9a-fA-F]+", next='div'),
+        Tok("onum",         r"0[0-7]+"),
         Tok("dnum",         r"""
                             (   (0|[1-9][0-9]*)         # DecimalIntegerLiteral
                                 \.                      # dot
         (r"\u1234 abc\u0020 \u0065_\u0067", [r"id \u1234", r"id abc\u0020", r"id \u0065_\u0067"]),
         ("123 1.234 0.123e-3 0 1E+40 1e1 .123", ["dnum 123", "dnum 1.234", "dnum 0.123e-3", "dnum 0", "dnum 1E+40", "dnum 1e1", "dnum .123"]),
         ("0x1 0xabCD 0XABcd", ["hnum 0x1", "hnum 0xabCD", "hnum 0XABcd"]),
+        ("010 0377 090", ["onum 010", "onum 0377", "dnum 0", "dnum 90"]),
         ("0xa123ghi", ["hnum 0xa123", "id ghi"]),
         ("function Function FUNCTION", ["keyword function", "id Function", "id FUNCTION"]),
         ("a//b", ["id a", "linecomment //b"]),
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.