1. Al Macmillan
  2. jslex


Ned Batchelder  committed 05fe9b3

Keywords have to match entire words.

  • Participants
  • Parent commits d6c6c11
  • Branches default

Comments (0)

Files changed (2)

File jslex.py

View file
  • Ignore whitespace
         self.regex = regex
         self.next = next
-def literals(choices):
-    """Create a regex from a space-separated list of literal `choices`."""
-    return "|".join(re.escape(c) for c in choices.split())
+def literals(choices, prefix="", suffix=""):
+    """Create a regex from a space-separated list of literal `choices`.
+    If provided, `prefix` and `suffix` will be attached to each choice
+    individually.
+    """
+    return "|".join(prefix+re.escape(c)+suffix for c in choices.split())
 class Lexer(object):
     """A generic multi-state regex-based lexer."""
                                 finally for function if import in instanceof new
                                 return super switch this throw try typeof var
                                 void while with
-                                """), next='reg'),
-        Tok("reserved",     literals("null true false"), next='div'),
+                                """, suffix=r"\b"), next='reg'),
+        Tok("reserved",     literals("null true false", suffix=r"\b"), next='div'),
         Tok("id",           r"([a-zA-Z_$]|\\u[0-9a-fA-Z]{4})([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})*", next='div'),
         Tok("hnum",         r"0[xX][0-9a-fA-F]+", next='div'),
         Tok("onum",         r"0[0-7]+"),

File tests.py

View file
  • Ignore whitespace
         ("0xa123ghi", ["hnum 0xa123", "id ghi"]),
         # keywords
         ("function Function FUNCTION", ["keyword function", "id Function", "id FUNCTION"]),
+        ("const constructor in inherits", ["keyword const", "id constructor", "keyword in", "id inherits"]),
+        ("true true_enough", ["reserved true", "id true_enough"]),
         # strings
         (''' 'hello' "hello" ''', ["string 'hello'", 'string "hello"']),
         (r""" 'don\'t' "don\"t" '"' "'" '\'' "\"" """,