Commits

Kirill Simonov committed bab0035

Trying to make libyaml bindings faster.

Comments (0)

Files changed (1)

     yaml_get_version(&major, &minor, &patch)
     return (major, minor, patch)
 
+def test_scanner(data):
+    cdef yaml_parser_t *parser
+    cdef yaml_token_t *token
+    cdef int done
+    if PyString_CheckExact(data) == 0:
+        raise TypeError("string input required")
+    parser = yaml_parser_new()
+    if parser == NULL:
+        raise MemoryError
+    yaml_parser_set_input_string(parser, PyString_AS_STRING(data), PyString_GET_SIZE(data))
+    done = 0
+    while done == 0:
+        token = yaml_parser_get_token(parser)
+        if token == NULL:
+            raise MemoryError
+        if token.type == YAML_STREAM_END_TOKEN:
+            done = 1
+        yaml_token_delete(token)
+    yaml_parser_delete(parser)
+
 cdef class Scanner:
 
     cdef yaml_parser_t *parser
     cdef int eof
     cdef object stream
+    cdef yaml_token_t *cached_token
+    cdef object cached_obj
 
     def __init__(self, stream):
-        cdef char *input
-        cdef int size
         if hasattr(stream, 'read'):
             stream = stream.read()
         if PyUnicode_CheckExact(stream) != 0:
         yaml_parser_set_input_string(self.parser, PyString_AS_STRING(stream), PyString_GET_SIZE(stream))
         self.eof = 0
         self.stream = stream
+        self.cached_token = NULL
+        self.cached_obj = None
 
     def __dealloc__(self):
         if self.parser != NULL:
 
     def get_token(self):
         cdef yaml_token_t *token
+        if self.cached_token != NULL:
+            yaml_token_delete(yaml_parser_get_token(self.parser))
+            obj = self.cached_obj
+            self.cached_token = NULL
+            self.cached_obj = None
+            return obj
         if self.eof != 0:
             return None
         token = yaml_parser_get_token(self.parser)
 
     def peek_token(self):
         cdef yaml_token_t *token
+        if self.cached_token != NULL:
+            return self.cached_obj
         if self.eof != 0:
             return None
         token = yaml_parser_peek_token(self.parser)
-        return self._convert(token)
+        obj = self._convert(token)
+        if token.type == YAML_STREAM_END_TOKEN:
+            self.eof = 1
+        self.cached_token = token
+        self.cached_obj = obj
+        return obj
 
     def check_token(self, *choices):
         cdef yaml_token_t *token
-        if self.eof != 0:
+        if self.cached_token != NULL:
+            obj = self.cached_obj
+        elif self.eof != 0:
             return False
-        token = yaml_parser_peek_token(self.parser)
-        obj = self._convert(token)
+        else:
+            token = yaml_parser_peek_token(self.parser)
+            obj = self._convert(token)
+            if token.type == YAML_STREAM_END_TOKEN:
+                self.eof = 1
+            self.cached_token = token
+            self.cached_obj = obj
         if not choices:
             return True
         for choice in choices:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.