Kirill Simonov avatar Kirill Simonov committed e44368b

Use Cython if available; added Python 3 support to _yaml.pyx.

Comments (0)

Files changed (7)

 
 #include <yaml.h>
 
+#if PY_MAJOR_VERSION >= 3
+
+#define PyString_CheckExact PyBytes_CheckExact
+#define PyString_AS_STRING  PyBytes_AS_STRING
+#define PyString_GET_SIZE   PyBytes_GET_SIZE
+#define PyString_FromStringAndSize  PyBytes_FromStringAndSize
+
+#endif
     cdef object anchors
 
     def __init__(self, stream):
+        cdef is_readable
         if yaml_parser_initialize(&self.parser) == 0:
             raise MemoryError
         self.parsed_event.type = YAML_NO_EVENT
-        if hasattr(stream, 'read'):
+        is_readable = 1
+        try:
+            stream.read
+        except AttributeError:
+            is_readable = 0
+        if is_readable:
             self.stream = stream
             try:
                 self.stream_name = stream.name
         elif token.type == YAML_STREAM_START_TOKEN:
             encoding = None
             if token.data.stream_start.encoding == YAML_UTF8_ENCODING:
-                encoding = "utf-8"
+                encoding = u"utf-8"
             elif token.data.stream_start.encoding == YAML_UTF16LE_ENCODING:
-                encoding = "utf-16-le"
+                encoding = u"utf-16-le"
             elif token.data.stream_start.encoding == YAML_UTF16BE_ENCODING:
-                encoding = "utf-16-be"
+                encoding = u"utf-16-be"
             return StreamStartToken(start_mark, end_mark, encoding)
         elif token.type == YAML_STREAM_END_TOKEN:
             return StreamEndToken(start_mark, end_mark)
         elif token.type == YAML_VERSION_DIRECTIVE_TOKEN:
-            return DirectiveToken("YAML",
+            return DirectiveToken(u"YAML",
                     (token.data.version_directive.major,
                         token.data.version_directive.minor),
                     start_mark, end_mark)
         elif token.type == YAML_TAG_DIRECTIVE_TOKEN:
-            return DirectiveToken("TAG",
-                    (token.data.tag_directive.handle,
-                        token.data.tag_directive.prefix),
+            handle = PyUnicode_DecodeUTF8(token.data.tag_directive.handle,
+                    strlen(token.data.tag_directive.handle), 'strict')
+            prefix = PyUnicode_DecodeUTF8(token.data.tag_directive.prefix,
+                    strlen(token.data.tag_directive.prefix), 'strict')
+            return DirectiveToken(u"TAG", (handle, prefix),
                     start_mark, end_mark)
         elif token.type == YAML_DOCUMENT_START_TOKEN:
             return DocumentStartToken(start_mark, end_mark)
     cdef CParser parser
     parser = <CParser>data
     value = parser.stream.read(size)
+    if PyUnicode_CheckExact(value) != 0:
+        value = PyUnicode_AsUTF8String(value)
     if PyString_CheckExact(value) == 0:
         raise TypeError("a string value is expected")
     if PyString_GET_SIZE(value) > size:
     cdef object anchors
     cdef int last_alias_id
     cdef int closed
+    cdef int decode_output
 
     def __init__(self, stream, canonical=None, indent=None, width=None,
             allow_unicode=None, line_break=None, encoding=None,
         if yaml_emitter_initialize(&self.emitter) == 0:
             raise MemoryError
         self.stream = stream
+        self.decode_output = 1
+        try:
+            stream.encoding
+        except AttributeError:
+            self.decode_output = 0
         yaml_emitter_set_output(&self.emitter, output_handler, <void *>self)    
         if canonical is not None:
             yaml_emitter_set_canonical(&self.emitter, 1)
         if node in self.anchors:
             if self.anchors[node] is None:
                 self.last_alias_id = self.last_alias_id+1
-                self.anchors[node] = "id%03d" % self.last_alias_id
+                self.anchors[node] = u"id%03d" % self.last_alias_id
         else:
             self.anchors[node] = None
             node_class = node.__class__
         anchor_object = self.anchors[node]
         anchor = NULL
         if anchor_object is not None:
-            anchor = PyString_AS_STRING(anchor_object)
+            anchor = PyString_AS_STRING(PyUnicode_AsUTF8String(anchor_object))
         if node in self.serialized_nodes:
             if yaml_alias_event_initialize(&event, anchor) == 0:
                 raise MemoryError
 cdef int output_handler(void *data, char *buffer, int size) except 0:
     cdef CEmitter emitter
     emitter = <CEmitter>data
-    value = PyString_FromStringAndSize(buffer, size)
+    if emitter.decode_output == 0:
+        value = PyString_FromStringAndSize(buffer, size)
+    else:
+        value = PyUnicode_DecodeUTF8(buffer, size, 'strict')
     emitter.stream.write(value)
     return 1
 

lib3/yaml/__init__.py

-
-__version__ = '3.08'
-__with_libyaml__ = False
 
 from .error import *
 
 from .loader import *
 from .dumper import *
 
+__version__ = '3.08'
+try:
+    from .cyaml import *
+    __with_libyaml__ = True
+except ImportError:
+    __with_libyaml__ = False
+
 import io
 
 def scan(stream, Loader=Loader):

lib3/yaml/cyaml.py

+
+__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
+        'CBaseDumper', 'CSafeDumper', 'CDumper']
+
+from _yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        BaseConstructor.__init__(self)
+        BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        SafeConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        Constructor.__init__(self)
+        Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        SafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
     sys.modules['distutils.extension'].Extension = _Extension
     sys.modules['distutils.command.build_ext'].Extension = _Extension
 
-try:
-    from Pyrex.Distutils import Extension as _Extension
-    from Pyrex.Distutils import build_ext as _build_ext
-    with_pyrex = True
-except ImportError:
-    with_pyrex = False
+with_pyrex = None
+if sys.version_info[0] < 3:
+    try:
+        from Cython.Distutils.extension import Extension as _Extension
+        from Cython.Distutils import build_ext as _build_ext
+        with_pyrex = 'cython'
+    except ImportError:
+        try:
+            from Pyrex.Distutils import Extension as _Extension
+            from Pyrex.Distutils import build_ext as _build_ext
+            with_pyrex = 'pyrex'
+        except ImportError:
+            pass
 
 
 class Distribution(_Distribution):
         self.check_extensions_list(self.extensions)
         filenames = []
         for ext in self.extensions:
-            if with_pyrex:
+            if with_pyrex == 'pyrex':
                 self.pyrex_sources(ext.sources, ext)
+            elif with_pyrex == 'cython':
+                self.cython_sources(ext.sources, ext)
             for filename in ext.sources:
                 filenames.append(filename)
                 base = os.path.splitext(filename)[0]
                 with_ext = self.check_extension_availability(ext)
             if not with_ext:
                 continue
-            if with_pyrex:
+            if with_pyrex == 'pyrex':
                 ext.sources = self.pyrex_sources(ext.sources, ext)
+            elif with_pyrex == 'cython':
+                ext.sources = self.cython_sources(ext.sources, ext)
             self.build_extension(ext)
 
     def check_extension_availability(self, ext):
 
 if __name__ == '__main__':
 
-    if sys.version_info[0] < 3:
+    package_dir = {
+            '2': 'lib',
+    }
 
-        setup(
-            name=NAME,
-            version=VERSION,
-            description=DESCRIPTION,
-            long_description=LONG_DESCRIPTION,
-            author=AUTHOR,
-            author_email=AUTHOR_EMAIL,
-            license=LICENSE,
-            platforms=PLATFORMS,
-            url=URL,
-            download_url=DOWNLOAD_URL,
-            classifiers=CLASSIFIERS,
+    setup(
+        name=NAME,
+        version=VERSION,
+        description=DESCRIPTION,
+        long_description=LONG_DESCRIPTION,
+        author=AUTHOR,
+        author_email=AUTHOR_EMAIL,
+        license=LICENSE,
+        platforms=PLATFORMS,
+        url=URL,
+        download_url=DOWNLOAD_URL,
+        classifiers=CLASSIFIERS,
 
-            package_dir={'': 'lib'},
-            packages=['yaml'],
-            ext_modules=[
-                Extension('_yaml', ['ext/_yaml.pyx'],
-                    'libyaml', "LibYAML bindings", LIBYAML_CHECK,
-                    libraries=['yaml']),
-            ],
+        package_dir={'': {2: 'lib', 3: 'lib3'}[sys.version_info[0]]},
+        packages=['yaml'],
+        ext_modules=[
+            Extension('_yaml', ['ext/_yaml.pyx'],
+                'libyaml', "LibYAML bindings", LIBYAML_CHECK,
+                libraries=['yaml']),
+        ],
 
-            distclass=Distribution,
-            cmdclass={
-                'build_ext': build_ext,
-                'bdist_rpm': bdist_rpm,
-                'test': test,
-            },
-        )
+        distclass=Distribution,
 
-    else:
+        cmdclass={
+            'build_ext': build_ext,
+            'bdist_rpm': bdist_rpm,
+            'test': test,
+        },
+    )
 
-        setup(
-            name=NAME,
-            version=VERSION,
-            description=DESCRIPTION,
-            long_description=LONG_DESCRIPTION,
-            author=AUTHOR,
-            author_email=AUTHOR_EMAIL,
-            license=LICENSE,
-            platforms=PLATFORMS,
-            url=URL,
-            download_url=DOWNLOAD_URL,
-            classifiers=CLASSIFIERS,
-
-            package_dir={'': 'lib3'},
-            packages=['yaml'],
-
-            cmdclass={
-                'test': test,
-            },
-        )
-

tests/lib3/test_structure.py

             return tuple(yaml.Loader.construct_sequence(self, node))
         def construct_mapping(self, node):
             pairs = self.construct_pairs(node)
-            pairs.sort()
+            pairs.sort(key=(lambda i: str(i)))
             return pairs
         def construct_undefined(self, node):
             return self.construct_scalar(node)
             return tuple(yaml.CanonicalLoader.construct_sequence(self, node))
         def construct_mapping(self, node):
             pairs = self.construct_pairs(node)
-            pairs.sort()
+            pairs.sort(key=(lambda i: str(i)))
             return pairs
         def construct_undefined(self, node):
             return self.construct_scalar(node)

tests/lib3/test_yaml_ext.py

     for collection in collections:
         if not isinstance(collection, dict):
             collection = vars(collection)
-        keys = collection.keys()
-        keys.sort()
-        for key in keys:
+        for key in sorted(collection):
             value = collection[key]
             if isinstance(value, types.FunctionType) and hasattr(value, 'unittest'):
                 functions.append(wrap_ext_function(value))
     for function in functions:
-        assert function.unittest_name not in globals()
-        globals()[function.unittest_name] = function
+        assert function.__name__ not in globals()
+        globals()[function.__name__] = function
 
 import test_tokens, test_structure, test_errors, test_resolver, test_constructor,   \
         test_emitter, test_representer, test_recursive
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.