Commits

Tyler Wade committed fbbabe9

Fix _cffi_backend

Comments (0)

Files changed (8)

pypy/interpreter/test/test_utf8.py

     assert s.rsplit(' ', 2) == u.rsplit(' ', 2)
     assert s.rsplit('\n') == [s]
 
-def test_copy_to_wcharp():
+def test_copy_to_new_wcharp():
     s = build_utf8str()
     if sys.maxunicode < 0x10000 and rffi.sizeof(rffi.WCHAR_T) == 4:
         # The last character requires a surrogate pair on narrow builds and
         # so won't be converted correctly by rffi.wcharp2unicode
         s = s[:-1]
 
-    wcharp = s.copy_to_wcharp()
+    wcharp = s.copy_to_new_wcharp()
     u = rffi.wcharp2unicode(wcharp)
     rffi.free_wcharp(wcharp)
     assert s == u
 
+    with s.scoped_wcharp_copy():
+        assert s == u
+
 def test_from_wcharp():
     def check(u):
         wcharp = rffi.unicode2wcharp(u)

pypy/interpreter/utf8.py

             byte_pos -= 1
         return byte_pos
 
-    def copy_to_wcharp(self, track_allocation=True):
+    def copy_to_new_wcharp(self, track_allocation=True):
         length = len(self) + 1
         if rffi.sizeof(rffi.WCHAR_T) == 2:
             for c in self.codepoint_iter():
 
         array = lltype.malloc(WCHAR_INTP.TO, length, flavor='raw',
                               track_allocation=track_allocation)
+
+        self.copy_to_wcharp(array, 0, length)
+        array[length - 1] = wchar_rint(0)
+
+        array = rffi.cast(rffi.CWCHARP, array)
+        return array
+
+    def copy_to_wcharp(self, dst, dststart, length):
         from pypy.interpreter.utf8_codecs import create_surrogate_pair
 
         i = 0;
         for c in self.codepoint_iter():
+            if i == length:
+                break
+
             if rffi.sizeof(rffi.WCHAR_T) == 2:
                 c1, c2 = create_surrogate_pair(c)
-                array[i] = wchar_rint(c1)
+                dst[i + dststart] = wchar_rint(c1)
                 if c2:
                     i += 1
-                    array[i] = wchar_rint(c2)
+                    dst[i + dststart] = wchar_rint(c2)
             else:
-                array[i] = wchar_rint(c)
+                dst[i + dststart] = wchar_rint(c)
 
             i += 1
 
-        array[i] = wchar_rint(0)
-        array = rffi.cast(rffi.CWCHARP, array)
-        return array
+    def scoped_wcharp_copy(self):
+        return WCharContextManager(self)
 
     @staticmethod
     def from_wcharp(wcharp):
     def build(self):
         return Utf8Str(self._builder.build(), self._is_ascii)
 
+class WCharContextManager(object):
+    def __init__(self, str):
+        self.str = str
+    def __enter__(self):
+        self.data = self.str.copy_to_new_wcharp()
+        return self.data
+    def __exit__(self, *args):
+        rffi.free_wcharp(self.data)
+
 # _______________________________________________
 
 # iter.current is the current (ie the last returned) element

pypy/module/__pypy__/interp_builders.py

 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
+from pypy.interpreter.utf8 import Utf8Builder
+from rpython.rlib.rstring import StringBuilder
 from rpython.tool.sourcetools import func_with_new_name
 
 
     return W_Builder
 
 W_StringBuilder = create_builder("StringBuilder", str, StringBuilder)
-W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder)
+W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, Utf8Builder)

pypy/module/_cffi_backend/ctypeprim.py

 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 
+from pypy.interpreter import utf8
 from pypy.interpreter.error import oefmt
+from pypy.interpreter.utf8 import Utf8Str, utf8ord
 from pypy.module._cffi_backend import cdataobj, misc
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return utf8ord(s)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
     _attrs_ = []
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.wrap(ord(unichardata[0]))
+        unichardata = rffi.cast(utf8.WCHAR_INTP, cdata)
+        return self.space.wrap(intmask(unichardata[0]))
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
+        s = Utf8Str.from_wcharpsize(unichardata, 1)
         return self.space.wrap(s)
 
     def string(self, cdataobj, maxlen):
 
     def convert_from_object(self, cdata, w_ob):
         value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        rffi.cast(utf8.WCHAR_INTP, cdata)[0] = utf8.wchar_rint(utf8ord(value))
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):

pypy/module/_cffi_backend/ctypeptr.py

 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
 
+from pypy.interpreter import utf8
+from pypy.interpreter.utf8 import Utf8Str
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
 from pypy.module._cffi_backend.ctypeobj import W_CType
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
+
+            unichardata = rffi.cast(utf8.WCHAR_INTP, cdata)
+            s.copy_to_wcharp(unichardata, 0, n)
             if n != self.length:
-                unichardata[n] = u'\x00'
+                unichardata[n] = utf8.wchar_rint(0)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
             if self.is_unichar_ptr_or_array():
                 cdata = rffi.cast(rffi.CWCHARP, cdata)
                 if length < 0:
-                    u = rffi.wcharp2unicode(cdata)
+                    u = Utf8Str.from_wcharp(cdata)
                 else:
-                    u = rffi.wcharp2unicoden(cdata, length)
+                    u = Utf8Str.from_wcharpn(cdata, length)
                 keepalive_until_here(cdataobj)
                 return space.wrap(u)
         #

pypy/module/_multibytecodec/c_codecs.py

         assert errorcb
         replace, end = errorcb(errors, namecb, reason,
                                stringdata, start, end)
-    inbuf = replace.copy_to_wcharp()
+    inbuf = replace.copy_to_new_wcharp()
     try:
         r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
     finally:
 def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None,
              namecb=None, ignore_error=0):
     inleft = len(unicodedata)
-    inbuf = unicodedata.copy_to_wcharp()
+    inbuf = unicodedata.copy_to_new_wcharp()
     try:
         if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
             raise MemoryError

pypy/module/_rawffi/alt/interp_funcptr.py

         self.argchain.arg(addr)
 
     def handle_unichar_p(self, w_ffitype, w_obj, unicodeval):
-        buf = unicodeval.copy_to_wcharp()
+        buf = unicodeval.copy_to_new_wcharp()
         self.w_func.to_free.append(rffi.cast(rffi.VOIDP, buf))
         addr = rffi.cast(rffi.ULONG, buf)
         self.argchain.arg(addr)

pypy/module/cpyext/unicodeobject.py

         # Copy unicode buffer
         w_unicode = from_ref(space, ref)
         u = space.unicode_w(w_unicode)
-        ref_unicode.c_buffer = u.copy_to_wcharp()
+        ref_unicode.c_buffer = u.copy_to_new_wcharp()
     return ref_unicode.c_buffer
 
 @cpython_api([PyObject], rffi.CWCHARP)