Commits

Amaury Forgeot d'Arc committed 26d73ca

cpyext: Implement the various FileSystem encoding functions.

  • Participants
  • Parent commits 8cf90cc
  • Branches py3k

Comments (0)

Files changed (5)

pypy/module/cpyext/api.py

 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
 Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE
+Py_CLEANUP_SUPPORTED
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))

pypy/module/cpyext/stubs.py

     raise NotImplementedError
     
 
-@cpython_api([PyObject], PyObject)
-def PyBytes_FromObject(space, o):
-    """Return the bytes representation of object o that implements the buffer
-    protocol."""
-    raise NotImplementedError
-    
-
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCell_Check(space, ob):
     """Return true if ob is a cell object; ob must not be NULL."""
     raise NotImplementedError
     
 
-@cpython_api([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-def PyUnicode_FSConverter(space, obj, result):
-    """ParseTuple converter: encode str objects to bytes using
-    PyUnicode_EncodeFSDefault(); bytes objects are output as-is.
-    result must be a PyBytesObject* which must be released when it is
-    no longer used.
-    """
-    raise NotImplementedError
-    
-
-@cpython_api([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-def PyUnicode_FSDecoder(space, obj, result):
-    """ParseTuple converter: decode bytes objects to str using
-    PyUnicode_DecodeFSDefaultAndSize(); str objects are output
-    as-is. result must be a PyUnicodeObject* which must be released
-    when it is no longer used.
-    """
-    raise NotImplementedError
-    
-
-@cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
-def PyUnicode_DecodeFSDefaultAndSize(space, s, size):
-    """Decode a string using Py_FileSystemDefaultEncoding and the
-    'surrogateescape' error handler, or 'strict' on Windows.
-    
-    If Py_FileSystemDefaultEncoding is not set, fall back to the
-    locale encoding.
-    
-    Use 'strict' error handler on Windows."""
-    raise NotImplementedError
-    
-
-@cpython_api([rffi.CCHARP], PyObject)
-def PyUnicode_DecodeFSDefault(space, s):
-    """Decode a null-terminated string using Py_FileSystemDefaultEncoding
-    and the 'surrogateescape' error handler, or 'strict' on Windows.
-    
-    If Py_FileSystemDefaultEncoding is not set, fall back to the
-    locale encoding.
-    
-    Use PyUnicode_DecodeFSDefaultAndSize() if you know the string length.
-    
-    Use 'strict' error handler on Windows."""
-    raise NotImplementedError
-    
-
-@cpython_api([PyObject], PyObject)
-def PyUnicode_EncodeFSDefault(space, unicode):
-    """Encode a Unicode object to Py_FileSystemDefaultEncoding with the
-    'surrogateescape' error handler, or 'strict' on Windows, and return
-    bytes. Note that the resulting bytes object may contain
-    null bytes.
-    
-    If Py_FileSystemDefaultEncoding is not set, fall back to the
-    locale encoding.
-    """
-    raise NotImplementedError
-    
-
 @cpython_api([rffi.CArrayPtr(Py_UNICODE), Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyUnicode_Encode(space, s, size, encoding, errors):
     """Encode the Py_UNICODE buffer s of the given size and return a Python

pypy/module/cpyext/test/test_unicodeobject.py

 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.unicodeobject import (
     Py_UNICODE, PyUnicodeObject, new_empty_unicode)
-from pypy.module.cpyext.api import PyObjectP, PyObject
+from pypy.module.cpyext.api import PyObjectP, PyObject, Py_CLEANUP_SUPPORTED
 from pypy.module.cpyext.pyobject import Py_DecRef, from_ref
 from pypy.rpython.lltypesystem import rffi, lltype
 import sys, py
         assert res == 0
         assert s == "12ሴ"
 
+    def test_encode_fsdefault(self, space, api):
+        w_u = space.wrap(u'sp�m')
+        w_s = api.PyUnicode_EncodeFSDefault(w_u)
+        with rffi.scoped_str2charp(space.str_w(w_s)) as encoded:
+            w_decoded = api.PyUnicode_DecodeFSDefaultAndSize(encoded, space.len_w(w_s))
+            assert space.eq_w(w_decoded, w_u)
+            w_decoded = api.PyUnicode_DecodeFSDefault(encoded)
+            assert space.eq_w(w_decoded, w_u)
+
+    def test_fsconverter(self, space, api):
+        # Input is bytes
+        w_input = space.wrapbytes("test")
+        with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
+            # Decoder
+            ret = api.PyUnicode_FSDecoder(w_input, result)
+            assert ret == Py_CLEANUP_SUPPORTED
+            assert space.isinstance_w(from_ref(space, result[0]), space.w_unicode)
+            assert api.PyUnicode_FSDecoder(None, result) == 1
+            # Converter
+            ret = api.PyUnicode_FSConverter(w_input, result)
+            assert ret == Py_CLEANUP_SUPPORTED
+            assert space.eq_w(from_ref(space, result[0]), w_input)
+            assert api.PyUnicode_FSDecoder(None, result) == 1
+        # Input is unicode
+        w_input = space.wrap("test")
+        with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
+            # Decoder
+            ret = api.PyUnicode_FSDecoder(w_input, result)
+            assert ret == Py_CLEANUP_SUPPORTED
+            assert space.eq_w(from_ref(space, result[0]), w_input)
+            assert api.PyUnicode_FSDecoder(None, result) == 1
+            # Converter
+            ret = api.PyUnicode_FSConverter(w_input, result)
+            assert ret == Py_CLEANUP_SUPPORTED
+            assert space.isinstance_w(from_ref(space, result[0]), space.w_bytes)
+            assert api.PyUnicode_FSDecoder(None, result) == 1
 
     def test_IS(self, space, api):
         for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f,

pypy/module/cpyext/unicodeobject.py

 from pypy.module.cpyext.api import (
     CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
     bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
-    CONST_WSTRING)
+    CONST_WSTRING, Py_CLEANUP_SUPPORTED)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr)
 from pypy.module.cpyext.stringobject import PyString_Check
+from pypy.module.cpyext.bytesobject import PyBytes_FromObject
 from pypy.module._codecs.interp_codecs import CodecState
+from pypy.module.posix.interp_posix import fsencode, fsdecode
 from pypy.objspace.std import unicodeobject, unicodetype, stringtype
 from pypy.rlib import runicode
 from pypy.tool.sourcetools import func_renamer
                              space.wrap("decoding Unicode is not supported"))
     return space.call_function(w_meth, w_encoding, w_errors)
 
+
+@cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0)
+def PyUnicode_FSConverter(space, w_obj, result):
+    """ParseTuple converter: encode str objects to bytes using
+    PyUnicode_EncodeFSDefault(); bytes objects are output as-is.
+    result must be a PyBytesObject* which must be released when it is
+    no longer used.
+    """
+    if not w_obj:
+        # Implement ParseTuple cleanup support
+        Py_DecRef(space, result[0])
+        return 1
+    if space.isinstance_w(w_obj, space.w_bytes):
+        w_output = w_obj
+    else:
+        w_obj = PyUnicode_FromObject(space, w_obj)
+        w_output = fsencode(space, w_obj)
+        if not space.isinstance_w(w_output, space.w_bytes):
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("encoder failed to return bytes"))
+    data = space.bytes0_w(w_output)  # Check for NUL bytes
+    result[0] = make_ref(space, w_output)
+    return Py_CLEANUP_SUPPORTED
+
+
+@cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0)
+def PyUnicode_FSDecoder(space, w_obj, result):
+    """ParseTuple converter: decode bytes objects to str using
+    PyUnicode_DecodeFSDefaultAndSize(); str objects are output
+    as-is. result must be a PyUnicodeObject* which must be released
+    when it is no longer used.
+    """
+    if not w_obj:
+        # Implement ParseTuple cleanup support
+        Py_DecRef(space, result[0])
+        return 1
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_output = w_obj
+    else:
+        w_obj = PyBytes_FromObject(space, w_obj)
+        w_output = fsdecode(space, w_obj)
+        if not space.isinstance_w(w_output, space.w_unicode):
+            raise OperationError(space.w_TypeError,
+                                 space.wrap("decoder failed to return unicode"))
+    data = space.unicode0_w(w_output)  # Check for NUL bytes
+    result[0] = make_ref(space, w_output)
+    return Py_CLEANUP_SUPPORTED
+    
+
+@cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
+def PyUnicode_DecodeFSDefaultAndSize(space, s, size):
+    """Decode a string using Py_FileSystemDefaultEncoding and the
+    'surrogateescape' error handler, or 'strict' on Windows.
+    
+    If Py_FileSystemDefaultEncoding is not set, fall back to the
+    locale encoding.
+    
+    Use 'strict' error handler on Windows."""
+    w_bytes = space.wrapbytes(rffi.charpsize2str(s, size))
+    return fsdecode(space, w_bytes)
+
+
+@cpython_api([rffi.CCHARP], PyObject)
+def PyUnicode_DecodeFSDefault(space, s):
+    """Decode a null-terminated string using Py_FileSystemDefaultEncoding
+    and the 'surrogateescape' error handler, or 'strict' on Windows.
+    
+    If Py_FileSystemDefaultEncoding is not set, fall back to the
+    locale encoding.
+    
+    Use PyUnicode_DecodeFSDefaultAndSize() if you know the string length.
+    
+    Use 'strict' error handler on Windows."""
+    w_bytes = space.wrapbytes(rffi.charp2str(s))
+    return fsdecode(space, w_bytes)
+
+
+@cpython_api([PyObject], PyObject)
+def PyUnicode_EncodeFSDefault(space, w_unicode):
+    """Encode a Unicode object to Py_FileSystemDefaultEncoding with the
+    'surrogateescape' error handler, or 'strict' on Windows, and return
+    bytes. Note that the resulting bytes object may contain
+    null bytes.
+    
+    If Py_FileSystemDefaultEncoding is not set, fall back to the
+    locale encoding.
+    """
+    return fsencode(space, w_unicode)
+
+
 @cpython_api([CONST_STRING], PyObject)
 def PyUnicode_FromString(space, s):
     """Create a Unicode object from an UTF-8 encoded null-terminated char buffer"""

pypy/module/posix/interp_posix.py

 
 def fsencode_w(space, w_obj):
     if space.isinstance_w(w_obj, space.w_unicode):
-        w_obj = space.call_method(w_obj, 'encode',
-                                  getfilesystemencoding(space),
-                                  space.wrap('surrogateescape'))
+        w_obj = fsencode(space, w_obj)
     return space.bytes0_w(w_obj)
 
+def fsencode(space, w_obj):
+    w_bytes = space.call_method(w_obj, 'encode',
+                                getfilesystemencoding(space),
+                                space.wrap('surrogateescape'))
+    return w_bytes
+
 def fsdecode(space, w_obj):
     w_unicode = space.call_method(w_obj, 'decode',
                                   getfilesystemencoding(space),