Source

pypy / pypy / interpreter / pyparser / pytokenize.py

Diff from to

pypy/interpreter/pyparser/pytokenize.py

 
 accepts = [True, True, True, True, True, True, True, True,
            True, True, False, True, True, True, True, False,
-           False, False, True, True, True, False, True,
-           False, True, False, True, False, False, True,
-           False, False, False, False, True, False, False,
-           False, True]
+           False, False, True, False, False, True, False,
+           False, True, False, True, False, True, False,
+           False, True, False, False, True, True, True,
+           False, False, True, False, False, False, True]
 states = [
     # 0
     {'\t': 0, '\n': 13, '\x0c': 0,
      '4': 5, '5': 5, '6': 5, '7': 5,
      '8': 5, '9': 5, ':': 13, ';': 13,
      '<': 9, '=': 12, '>': 8, '@': 13,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'A': 1, 'B': 2, 'C': 1, 'D': 1,
      'E': 1, 'F': 1, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 2, 'S': 1, 'T': 1,
-     'U': 3, 'V': 1, 'W': 1, 'X': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 2, 'V': 1, 'W': 1, 'X': 1,
      'Y': 1, 'Z': 1, '[': 13, '\\': 17,
      ']': 13, '^': 12, '_': 1, '`': 13,
-     'a': 1, 'b': 1, 'c': 1, 'd': 1,
+     'a': 1, 'b': 2, 'c': 1, 'd': 1,
      'e': 1, 'f': 1, 'g': 1, 'h': 1,
      'i': 1, 'j': 1, 'k': 1, 'l': 1,
      'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 2, 's': 1, 't': 1,
-     'u': 3, 'v': 1, 'w': 1, 'x': 1,
+     'q': 1, 'r': 3, 's': 1, 't': 1,
+     'u': 2, 'v': 1, 'w': 1, 'x': 1,
      'y': 1, 'z': 1, '{': 13, '|': 12,
      '}': 13, '~': 13},
     # 1
      'E': 1, 'F': 1, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 1, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 3, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1},
+    # 3
+    {'"': 16, "'": 15, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 1, 'S': 1, 'T': 1,
      'U': 1, 'V': 1, 'W': 1, 'X': 1,
      'Y': 1, 'Z': 1, '_': 1, 'a': 1,
      'r': 1, 's': 1, 't': 1, 'u': 1,
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1},
-    # 3
-    {'"': 16, "'": 15, '0': 1, '1': 1,
-     '2': 1, '3': 1, '4': 1, '5': 1,
-     '6': 1, '7': 1, '8': 1, '9': 1,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 2, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
-     'b': 1, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
-     'j': 1, 'k': 1, 'l': 1, 'm': 1,
-     'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 2, 's': 1, 't': 1, 'u': 1,
-     'v': 1, 'w': 1, 'x': 1, 'y': 1,
-     'z': 1},
     # 4
-    {'.': 22, '0': 20, '1': 20, '2': 20,
-     '3': 20, '4': 20, '5': 20, '6': 20,
-     '7': 20, '8': 21, '9': 21, 'E': 23,
-     'J': 13, 'L': 13, 'X': 19, 'e': 23,
-     'j': 13, 'l': 13, 'x': 19},
+    {'.': 24, '0': 21, '1': 21, '2': 21,
+     '3': 21, '4': 21, '5': 21, '6': 21,
+     '7': 21, '8': 23, '9': 23, 'B': 22,
+     'E': 25, 'J': 13, 'L': 13, 'O': 20,
+     'X': 19, 'b': 22, 'e': 25, 'j': 13,
+     'l': 13, 'o': 20, 'x': 19},
     # 5
-    {'.': 22, '0': 5, '1': 5, '2': 5,
+    {'.': 24, '0': 5, '1': 5, '2': 5,
      '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 23,
-     'J': 13, 'L': 13, 'e': 23, 'j': 13,
+     '7': 5, '8': 5, '9': 5, 'E': 25,
+     'J': 13, 'L': 13, 'e': 25, 'j': 13,
      'l': 13},
     # 6
-    {'0': 24, '1': 24, '2': 24, '3': 24,
-     '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24},
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26},
     # 7
     {'*': 12, '=': 13},
     # 8
     # 14
     {'\n': 13},
     # 15
-    {automata.DEFAULT: 28, '\n': 25,
-     '\r': 25, "'": 26, '\\': 27},
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 28, '\\': 29},
     # 16
-    {automata.DEFAULT: 31, '\n': 25,
-     '\r': 25, '"': 29, '\\': 30},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 31, '\\': 32},
     # 17
     {'\n': 13, '\r': 14},
     # 18
-    {automata.DEFAULT: 18, '\n': 25, '\r': 25},
+    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
     # 19
-    {'0': 19, '1': 19, '2': 19, '3': 19,
-     '4': 19, '5': 19, '6': 19, '7': 19,
-     '8': 19, '9': 19, 'A': 19, 'B': 19,
-     'C': 19, 'D': 19, 'E': 19, 'F': 19,
-     'L': 13, 'a': 19, 'b': 19, 'c': 19,
-     'd': 19, 'e': 19, 'f': 19, 'l': 13},
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
     # 20
-    {'.': 22, '0': 20, '1': 20, '2': 20,
-     '3': 20, '4': 20, '5': 20, '6': 20,
-     '7': 20, '8': 21, '9': 21, 'E': 23,
-     'J': 13, 'L': 13, 'e': 23, 'j': 13,
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
+    # 21
+    {'.': 24, '0': 21, '1': 21, '2': 21,
+     '3': 21, '4': 21, '5': 21, '6': 21,
+     '7': 21, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'L': 13, 'e': 25, 'j': 13,
      'l': 13},
-    # 21
-    {'.': 22, '0': 21, '1': 21, '2': 21,
-     '3': 21, '4': 21, '5': 21, '6': 21,
-     '7': 21, '8': 21, '9': 21, 'E': 23,
-     'J': 13, 'e': 23, 'j': 13},
     # 22
-    {'0': 22, '1': 22, '2': 22, '3': 22,
-     '4': 22, '5': 22, '6': 22, '7': 22,
-     '8': 22, '9': 22, 'E': 32, 'J': 13,
-     'e': 32, 'j': 13},
+    {'0': 36, '1': 36},
     # 23
-    {'+': 33, '-': 33, '0': 34, '1': 34,
-     '2': 34, '3': 34, '4': 34, '5': 34,
-     '6': 34, '7': 34, '8': 34, '9': 34},
+    {'.': 24, '0': 23, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
     # 24
     {'0': 24, '1': 24, '2': 24, '3': 24,
      '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24, 'E': 32, 'J': 13,
-     'e': 32, 'j': 13},
+     '8': 24, '9': 24, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
     # 25
+    {'+': 38, '-': 38, '0': 39, '1': 39,
+     '2': 39, '3': 39, '4': 39, '5': 39,
+     '6': 39, '7': 39, '8': 39, '9': 39},
+    # 26
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 27
     {},
-    # 26
+    # 28
     {"'": 13},
-    # 27
-    {automata.DEFAULT: 35, '\n': 13, '\r': 14},
-    # 28
-    {automata.DEFAULT: 28, '\n': 25,
-     '\r': 25, "'": 13, '\\': 27},
     # 29
+    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
+    # 30
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 31
     {'"': 13},
-    # 30
-    {automata.DEFAULT: 36, '\n': 13, '\r': 14},
-    # 31
-    {automata.DEFAULT: 31, '\n': 25,
-     '\r': 25, '"': 13, '\\': 30},
     # 32
-    {'+': 37, '-': 37, '0': 38, '1': 38,
-     '2': 38, '3': 38, '4': 38, '5': 38,
-     '6': 38, '7': 38, '8': 38, '9': 38},
+    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
     # 33
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34},
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
     # 34
     {'0': 34, '1': 34, '2': 34, '3': 34,
      '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'J': 13, 'j': 13},
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'L': 13, 'a': 34, 'b': 34, 'c': 34,
+     'd': 34, 'e': 34, 'f': 34, 'l': 13},
     # 35
-    {automata.DEFAULT: 35, '\n': 25,
-     '\r': 25, "'": 13, '\\': 27},
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35,
+     'L': 13, 'l': 13},
     # 36
-    {automata.DEFAULT: 36, '\n': 25,
-     '\r': 25, '"': 13, '\\': 30},
+    {'0': 36, '1': 36, 'L': 13, 'l': 13},
     # 37
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38},
+    {'+': 42, '-': 42, '0': 43, '1': 43,
+     '2': 43, '3': 43, '4': 43, '5': 43,
+     '6': 43, '7': 43, '8': 43, '9': 43},
     # 38
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38,
-     '8': 38, '9': 38, 'J': 13, 'j': 13},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39},
+    # 39
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'J': 13, 'j': 13},
+    # 40
+    {automata.DEFAULT: 40, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 41
+    {automata.DEFAULT: 41, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
+    # 42
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43},
+    # 43
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43, 'J': 13, 'j': 13},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 
 
 endDFAs = {"'" : singleDFA,
            '"' : doubleDFA,
-           "r" : None,
-           "R" : None,
-           "u" : None,
-           "U" : None}
+           'r' : None,
+           'R' : None,
+           'u' : None,
+           'U' : None,
+           'b' : None,
+           'B' : None}
 
-for uniPrefix in ("", "u", "U"):
+for uniPrefix in ("", "u", "U", "b", "B"):
     for rawPrefix in ("", "r", "R"):
         prefix = uniPrefix + rawPrefix
         endDFAs[prefix + "'''"] = single3DFA
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
+          "b'''", 'b"""', "B'''", 'B"""',
           "ur'''", 'ur"""', "Ur'''", 'Ur"""',
-          "uR'''", 'uR"""', "UR'''", 'UR"""'):
+          "uR'''", 'uR"""', "UR'''", 'UR"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""'):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
+          "b'", 'b"', "B'", 'B"',
           "ur'", 'ur"', "Ur'", 'Ur"',
-          "uR'", 'uR"', "UR'", 'UR"' ):
+          "uR'", 'uR"', "UR'", 'UR"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"'):
     single_quoted[t] = t
 
 tabsize = 8
 
 # ______________________________________________________________________
 # End of pytokenize.py
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.