Commits

Matt Bone committed f5d6221

tokenize yields all tokenize but whitespace

Comments (0)

Files changed (1)

in_py/parse_sexp.py

 open_paren_re = re.compile('\(')
 close_paren_re = re.compile('\)')
 symbol_start_re = re.compile('[a-zA-Z\+\-]')
-symbol_internal_re = re.compile('[a-zA-Z\+\-1-9]')
 number_re = re.compile('[0-9]')
 whitespace_re = re.compile(' ')
 
     close_paren_re: close_paren_token,
     open_paren_re: open_paren_token,
     symbol_start_re: symbol_token,
-    #symbol_internal_re:symbol_token,
     number_re:number_token,
     whitespace_re: whitespace_token,
     }
  
 
 def tokenize(some_string):
-    pass
+    for token in _tokenize(some_string):
+        if not isinstance(token, whitespace_token):
+            yield token
+
 
 def sexp_to_tree(sexp):
     pass
         self.assertEqual(close.value, ")")
 
 
+    def test_tokenize(self):
+        tokens = list(tokenize("(ab9+1-  999)"))
+        self.assertEqual(len(tokens), 4)
+
+        open = tokens[0]
+        sym = tokens[1]
+        num = tokens[2]
+        close = tokens[3]        
+
+        self.assertTrue(isinstance(open, open_paren_token))
+        self.assertEqual(open.value, "(")
+        
+        self.assertTrue(isinstance(sym, symbol_token))
+        self.assertEqual(sym.value, "ab9+1-")
+
+        self.assertTrue(isinstance(num, number_token))
+        self.assertEqual(num.value, "999")
+                
+        self.assertTrue(isinstance(close, close_paren_token))
+        self.assertEqual(close.value, ")")
+
+
     def test_symbol_number_error(self):
         self.failUnlessRaises(syntax_exception, lambda: list(_tokenize("9abc")))
         self.failUnlessRaises(syntax_exception, lambda: list(_tokenize("989abc")))
         self.assertTrue(symbol_start_re.match('M') is not None)        
 
 
-    def test_symbol_internal(self):
-        self.assertTrue(symbol_internal_re.match(')') is None)
-        self.assertTrue(symbol_internal_re.match('(') is None)
-        self.assertTrue(symbol_internal_re.match('+') is not None)
-        self.assertTrue(symbol_internal_re.match('9') is not None)
-        self.assertTrue(symbol_internal_re.match('a') is not None)
-        self.assertTrue(symbol_internal_re.match('A') is not None)        
-        self.assertTrue(symbol_internal_re.match('m') is not None)
-        self.assertTrue(symbol_internal_re.match('M') is not None)        
-
-
     def test_number(self):
         self.assertTrue(number_re.match(')') is None)
         self.assertTrue(number_re.match('(') is None)