Source

pypy / pypy / interpreter / pyparser / pytoken.py

Diff from to

pypy/interpreter/pyparser/pytoken.py

-# A replacement for the token module
-#
-# adds a new map token_values to avoid doing getattr on the module
-# from PyPy RPython
+"""Python token definitions."""
 
-N_TOKENS = 0
+python_tokens = {}
+python_opmap = {}
 
-tok_name = {}
-tok_values = {}
+def _add_tok(name, *values):
+    index = len(python_tokens)
+    assert index < 256
+    python_tokens[name] = index
+    for value in values:
+        python_opmap[value] = index
 
-def add_token(name, value=None):
-    global N_TOKENS
-    if value is None:
-        value = N_TOKENS
-        N_TOKENS += 1
-    _g = globals()
-    _g[name] = value
-    tok_name[value] = name
-    tok_values[name] = value
-
-# This is used to replace None
-add_token( 'NULLTOKEN', -1 )
-
-# For compatibility, this produces the same constant values as Python 2.4.
-add_token( 'ENDMARKER' )
-add_token( 'NAME' )
-add_token( 'NUMBER' )
-add_token( 'STRING' )
-add_token( 'NEWLINE' )
-add_token( 'INDENT' )
-add_token( 'DEDENT' )
-add_token( 'LPAR' )
-add_token( 'RPAR' )
-add_token( 'LSQB' )
-add_token( 'RSQB' )
-add_token( 'COLON' )
-add_token( 'COMMA' )
-add_token( 'SEMI' )
-add_token( 'PLUS' )
-add_token( 'MINUS' )
-add_token( 'STAR' )
-add_token( 'SLASH' )
-add_token( 'VBAR' )
-add_token( 'AMPER' )
-add_token( 'LESS' )
-add_token( 'GREATER' )
-add_token( 'EQUAL' )
-add_token( 'DOT' )
-add_token( 'PERCENT' )
-add_token( 'BACKQUOTE' )
-add_token( 'LBRACE' )
-add_token( 'RBRACE' )
-add_token( 'EQEQUAL' )
-add_token( 'NOTEQUAL' )
-add_token( 'LESSEQUAL' )
-add_token( 'GREATEREQUAL' )
-add_token( 'TILDE' )
-add_token( 'CIRCUMFLEX' )
-add_token( 'LEFTSHIFT' )
-add_token( 'RIGHTSHIFT' )
-add_token( 'DOUBLESTAR' )
-add_token( 'PLUSEQUAL' )
-add_token( 'MINEQUAL' )
-add_token( 'STAREQUAL' )
-add_token( 'SLASHEQUAL' )
-add_token( 'PERCENTEQUAL' )
-add_token( 'AMPEREQUAL' )
-add_token( 'VBAREQUAL' )
-add_token( 'CIRCUMFLEXEQUAL' )
-add_token( 'LEFTSHIFTEQUAL' )
-add_token( 'RIGHTSHIFTEQUAL' )
-add_token( 'DOUBLESTAREQUAL' )
-add_token( 'DOUBLESLASH' )
-add_token( 'DOUBLESLASHEQUAL' )
-add_token( 'AT' )
-add_token( 'OP' )
-add_token( 'ERRORTOKEN' )
+_add_tok('ENDMARKER')
+_add_tok('NAME')
+_add_tok('NUMBER')
+_add_tok('STRING')
+_add_tok('NEWLINE')
+_add_tok('INDENT')
+_add_tok('DEDENT')
+_add_tok('LPAR', "(")
+_add_tok('RPAR', ")")
+_add_tok('LSQB', "[")
+_add_tok('RSQB', "]")
+_add_tok('COLON', ":")
+_add_tok('COMMA',  "," )
+_add_tok('SEMI', ";" )
+_add_tok('PLUS', "+" )
+_add_tok('MINUS', "-" )
+_add_tok('STAR', "*" )
+_add_tok('SLASH', "/" )
+_add_tok('VBAR', "|" )
+_add_tok('AMPER', "&" )
+_add_tok('LESS', "<" )
+_add_tok('GREATER', ">" )
+_add_tok('EQUAL', "=" )
+_add_tok('DOT', "." )
+_add_tok('PERCENT', "%" )
+_add_tok('BACKQUOTE', "`" )
+_add_tok('LBRACE', "{" )
+_add_tok('RBRACE', "}" )
+_add_tok('EQEQUAL', "==" )
+_add_tok('NOTEQUAL', "!=", "<>" )
+_add_tok('LESSEQUAL', "<=" )
+_add_tok('GREATEREQUAL', ">=" )
+_add_tok('TILDE', "~" )
+_add_tok('CIRCUMFLEX', "^" )
+_add_tok('LEFTSHIFT', "<<" )
+_add_tok('RIGHTSHIFT', ">>" )
+_add_tok('DOUBLESTAR', "**" )
+_add_tok('PLUSEQUAL', "+=" )
+_add_tok('MINEQUAL', "-=" )
+_add_tok('STAREQUAL', "*=" )
+_add_tok('SLASHEQUAL', "/=" )
+_add_tok('PERCENTEQUAL', "%=" )
+_add_tok('AMPEREQUAL', "&=" )
+_add_tok('VBAREQUAL', "|=" )
+_add_tok('CIRCUMFLEXEQUAL', "^=" )
+_add_tok('LEFTSHIFTEQUAL', "<<=" )
+_add_tok('RIGHTSHIFTEQUAL', ">>=" )
+_add_tok('DOUBLESTAREQUAL', "**=" )
+_add_tok('DOUBLESLASH', "//" )
+_add_tok('DOUBLESLASHEQUAL',"//=" )
+_add_tok('AT', "@" )
+_add_tok('OP')
+_add_tok('ERRORTOKEN')
 
 # extra PyPy-specific tokens
-add_token( "COMMENT" )
-add_token( "NL" )
+_add_tok("COMMENT")
+_add_tok("NL")
 
-# a reverse mapping from internal tokens def to more pythonic tokens
-tok_punct = {
-    "&" : AMPER,
-    "&=" : AMPEREQUAL,
-    "`" : BACKQUOTE,
-    "^" : CIRCUMFLEX,
-    "^=" : CIRCUMFLEXEQUAL,
-    ":" : COLON,
-    "," : COMMA,
-    "." : DOT,
-    "//" : DOUBLESLASH,
-    "//=" : DOUBLESLASHEQUAL,
-    "**" : DOUBLESTAR,
-    "**=" : DOUBLESTAREQUAL,
-    "==" : EQEQUAL,
-    "=" : EQUAL,
-    ">" : GREATER,
-    ">=" : GREATEREQUAL,
-    "{" : LBRACE,
-    "}" : RBRACE,
-    "<<" : LEFTSHIFT,
-    "<<=" : LEFTSHIFTEQUAL,
-    "<" : LESS,
-    "<=" : LESSEQUAL,
-    "(" : LPAR,
-    "[" : LSQB,
-    "-=" : MINEQUAL,
-    "-" : MINUS,
-    "!=" : NOTEQUAL,
-    "<>" : NOTEQUAL,
-    "%" : PERCENT,
-    "%=" : PERCENTEQUAL,
-    "+" : PLUS,
-    "+=" : PLUSEQUAL,
-    ")" : RBRACE,
-    ">>" : RIGHTSHIFT,
-    ">>=" : RIGHTSHIFTEQUAL,
-    ")" : RPAR,
-    "]" : RSQB,
-    ";" : SEMI,
-    "/" : SLASH,
-    "/=" : SLASHEQUAL,
-    "*" : STAR,
-    "*=" : STAREQUAL,
-    "~" : TILDE,
-    "|" : VBAR,
-    "|=" : VBAREQUAL,
-    "@": AT,
-    }
-tok_rpunct = {}
-for string, value in tok_punct.items():
-    tok_rpunct[value] = string
-
+del _add_tok