Commits

Gigih Aji Ibrahim committed cdf03b0

fix longstading bugs in lexer: token #DEDENT not properly inserted

  • Participants
  • Parent commits 2480146

Comments (0)

Files changed (2)

File src/lexer4.py

             self.indentStack.append(self.column)
             return True
         elif self.column < lastIndent:
-            self.appendToken('#DEDENT', begin)
-            self.indentStack.pop()
+            while True:
+                self.appendToken('#DEDENT', begin)
+                self.indentStack.pop()
+                if self.column >= self.indentStack[-1]: break
             return True
 
         return False

File src/unittest/test_lexer4.py

                           '#ID',
                       '#DEDENT']
 
+def testIndent2():
+    ## some edge case
+    data = r"""
+class A:
+    def main:
+        if true:
+            a
+    def embuh:
+        c
+"""    
+    lexer = init(data)
+    tokens = getTokenContent(getAllToken(lexer))
+
+    assert tokens.count('#INDENT') == 4
+    assert tokens.count('#DEDENT') == 4
+    
 
 def testNumber():
     data = r"""10 10.23 0xFACEB00 1_0 10.234__567 0"""