Matt Bone avatar Matt Bone committed c025738

more tests, tokenizer is totally working

Comments (0)

Files changed (1)

in_py/parse_sexp.py

                 current_token = current_token + token
             except next_token_exception:
                 yield current_token
-                current_token = next_token
+                current_token = token
 
     yield current_token
  
         token = tokens[0]
         self.assertTrue(isinstance(token, number_token))
         self.assertEqual(token.value, "9876")
+
+
+    def test_complex_under_tokenize(self):
+        tokens = list(_tokenize("(ab9+1-  999)"))
+        self.assertEqual(len(tokens), 5)
+
+        open = tokens[0]
+        sym = tokens[1]
+        space = tokens[2]
+        num = tokens[3]
+        close = tokens[4]        
+
+        self.assertTrue(isinstance(open, open_paren_token))
+        self.assertEqual(open.value, "(")
+        
+        self.assertTrue(isinstance(sym, symbol_token))
+        self.assertEqual(sym.value, "ab9+1-")
+
+        self.assertTrue(isinstance(space, whitespace_token))
+        self.assertEqual(space.value, "  ")
+
+        self.assertTrue(isinstance(num, number_token))
+        self.assertEqual(num.value, "999")
+                
+        self.assertTrue(isinstance(close, close_paren_token))
+        self.assertEqual(close.value, ")")
+
+
+    def test_symbol_number_error(self):
+        self.failUnlessRaises(syntax_exception, lambda: list(_tokenize("9abc")))
+        self.failUnlessRaises(syntax_exception, lambda: list(_tokenize("989abc")))
         
 
 class TestTokens(unittest.TestCase):
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.