Kirill Simonov avatar Kirill Simonov committed 6c7c12b

Fix #11 (Thanks to edemaine(at)mit.edu).

Comments (0)

Files changed (6)

lib/yaml/__init__.py

     """
     return dump_all([data], stream, Dumper=SafeDumper, **kwds)
 
-def add_implicit_detector(tag, regexp, first=None,
+def add_implicit_resolver(tag, regexp, first=None,
         Loader=Loader, Dumper=Dumper):
     """
     Add an implicit scalar detector.

lib/yaml/emitter.py

             if ch in u'\n\x85\u2028\u2029':
                 line_breaks = True
             if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'):
-                if ch < u'\x80' or ch == u'\uFEFF': # '\uFEFF' is BOM.
-                    special_characters = True
-                else:
+                if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF'
+                        or u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF':
                     unicode_characters = True
                     if not self.allow_unicode:
                         special_characters = True
+                else:
+                    special_characters = True
 
             # Spaces, line breaks, and how they are mixed. State machine.
 
                 ch = text[end]
             if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \
                     or not (u'\x20' <= ch <= u'\x7E'
-                            or (self.allow_unicode and ch > u'\x7F')):
+                        or (self.allow_unicode
+                            and (u'\xA0' <= ch <= u'\uD7FF'
+                                or u'\uE000' <= ch <= u'\uFFFD'))):
                 if start < end:
                     data = text[start:end]
                     self.column += len(data)

lib/yaml/nodes.py

         self.end_mark = end_mark
     def __repr__(self):
         value = self.value
-        if isinstance(value, list):
-            if len(value) == 0:
-                value = '<empty>'
-            elif len(value) == 1:
-                value = '<1 item>'
-            else:
-                value = '<%d items>' % len(value)
-        else:
-            if len(value) > 75:
-                value = repr(value[:70]+u' ... ')
-            else:
-                value = repr(value)
+        #if isinstance(value, list):
+        #    if len(value) == 0:
+        #        value = '<empty>'
+        #    elif len(value) == 1:
+        #        value = '<1 item>'
+        #    else:
+        #        value = '<%d items>' % len(value)
+        #else:
+        #    if len(value) > 75:
+        #        value = repr(value[:70]+u' ... ')
+        #    else:
+        #        value = repr(value)
+        value = repr(value)
         return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
 
 class ScalarNode(Node):

tests/data/emitting-unacceptable-unicode-character-bug.code

+u"\udd00"

tests/data/emitting-unacceptable-unicode-character-bug.data

+u"\udd00"

tests/test_representer.py

 
 class TestRepresenterTypes(test_appliance.TestAppliance):
 
-    def _testTypes(self, test_name, data_filename, code_filename):
+    def _testTypesUnicode(self, test_name, data_filename, code_filename):
+        return self._testTypes(test_name, data_filename, code_filename, allow_unicode=True)
+
+    def _testTypes(self, test_name, data_filename, code_filename, allow_unicode=False):
         data1 = eval(file(code_filename, 'rb').read())
         data2 = None
         output = None
         try:
-            output = dump(data1, Dumper=MyDumper)
+            output = dump(data1, Dumper=MyDumper, allow_unicode=allow_unicode)
             data2 = load(output, Loader=MyLoader)
             self.failUnlessEqual(type(data1), type(data2))
             try:
             raise
 
 TestRepresenterTypes.add_tests('testTypes', '.data', '.code')
+TestRepresenterTypes.add_tests('testTypesUnicode', '.data', '.code')
 
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.