Commits

Philip Jenvey committed b3e42a0

updates

  • Participants
  • Parent commits edbb3b3

Comments (0)

Files changed (3)

-boring

File multibytecodec

 # HG changeset patch
-# Parent b5bc2f812d764554599a7f6f9624b48e3170785b
+# Parent 54515dec1d201fb0a5a3926d970830aad10a92df
 add missing _multibytecodec stateful classes
 
-diff -r b5bc2f812d76 pypy/module/_multibytecodec/__init__.py
---- a/pypy/module/_multibytecodec/__init__.py	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/module/_multibytecodec/__init__.py	Tue Jul 26 10:14:42 2011 -0700
+diff -r 54515dec1d20 pypy/module/_multibytecodec/__init__.py
+--- a/pypy/module/_multibytecodec/__init__.py	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/module/_multibytecodec/__init__.py	Fri Jul 29 16:28:46 2011 -0700
 @@ -7,13 +7,22 @@
          # for compatibility this name is obscured, and should be called
          # via the _codecs_*.py modules written in lib_pypy.
          'MultibyteStreamReader':
              'app_multibytecodec.MultibyteStreamReader',
          'MultibyteStreamWriter':
-diff -r b5bc2f812d76 pypy/module/_multibytecodec/c_codecs.py
---- a/pypy/module/_multibytecodec/c_codecs.py	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/module/_multibytecodec/c_codecs.py	Tue Jul 26 10:14:42 2011 -0700
+diff -r 54515dec1d20 pypy/module/_multibytecodec/c_codecs.py
+--- a/pypy/module/_multibytecodec/c_codecs.py	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/module/_multibytecodec/c_codecs.py	Fri Jul 29 16:28:46 2011 -0700
 @@ -4,6 +4,8 @@
  from pypy.tool.autopath import pypydir
  
  
  
  class EncodeDecodeError(Exception):
-@@ -179,7 +181,7 @@
+@@ -57,6 +59,8 @@
+         "pypy_cjk_dec_inbuf_remaining", "pypy_cjk_dec_inbuf_consumed",
+         "pypy_cjk_dec_replace_on_error",
+ 
++        "pypy_cjk_dec_init_state", "pypy_cjk_dec_free_state",
++
+         "pypy_cjk_enc_init", "pypy_cjk_enc_free", "pypy_cjk_enc_chunk",
+         "pypy_cjk_enc_reset", "pypy_cjk_enc_outbuf", "pypy_cjk_enc_outlen",
+         "pypy_cjk_enc_inbuf_remaining", "pypy_cjk_enc_inbuf_consumed",
+@@ -72,6 +76,9 @@
+ MULTIBYTECODEC_P = rffi.COpaquePtr('struct MultibyteCodec_s',
+                                    compilation_info=eci)
+ 
++MULTIBYTECODECSTATE_P = rffi.COpaquePtr(typedef='MultibyteCodec_State',
++                                        compilation_info=eci)
++
+ def llexternal(*args, **kwds):
+     kwds.setdefault('compilation_info', eci)
+     kwds.setdefault('sandboxsafe', True)
+@@ -111,6 +118,11 @@
+                                            [DECODEBUF_P, rffi.CWCHARP,
+                                             rffi.SSIZE_T, rffi.SSIZE_T],
+                                            rffi.SSIZE_T)
++pypy_cjk_dec_init_state = llexternal('pypy_cjk_dec_init_state',
++                               [MULTIBYTECODEC_P],
++                               MULTIBYTECODECSTATE_P)
++pypy_cjk_dec_free_state = llexternal('pypy_cjk_dec_free_state',
++                                     [MULTIBYTECODECSTATE_P], lltype.Void)
+ 
+ def decode(codec, stringdata, errors="strict", errorcb=None, namecb=None):
+     inleft = len(stringdata)
+@@ -179,7 +191,7 @@
                                 ENCODEBUF_P)
  pypy_cjk_enc_free = llexternal('pypy_cjk_enc_free', [ENCODEBUF_P],
                                 lltype.Void)
                                  rffi.SSIZE_T)
  pypy_cjk_enc_reset = llexternal('pypy_cjk_enc_reset', [ENCODEBUF_P],
                                  rffi.SSIZE_T)
-@@ -196,7 +198,8 @@
+@@ -196,7 +208,8 @@
                                              rffi.SSIZE_T, rffi.SSIZE_T],
                                             rffi.SSIZE_T)
  
      inleft = len(unicodedata)
      inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
      try:
-@@ -205,8 +208,9 @@
+@@ -205,8 +218,9 @@
              raise MemoryError
          try:
              while True:
                      break
                  multibytecodec_encerror(encodebuf, r, errors,
                                          codec, errorcb, namecb, unicodedata)
-@@ -249,7 +253,9 @@
+@@ -249,7 +263,9 @@
          replace = ""
      elif errors == "replace":
          try:
          except EncodeDecodeError:
              replace = "?"
      else:
-diff -r b5bc2f812d76 pypy/module/_multibytecodec/interp_multibytecodec.py
---- a/pypy/module/_multibytecodec/interp_multibytecodec.py	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/module/_multibytecodec/interp_multibytecodec.py	Tue Jul 26 10:14:42 2011 -0700
+diff -r 54515dec1d20 pypy/module/_multibytecodec/interp_multibytecodec.py
+--- a/pypy/module/_multibytecodec/interp_multibytecodec.py	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/module/_multibytecodec/interp_multibytecodec.py	Fri Jul 29 16:28:46 2011 -0700
 @@ -1,9 +1,12 @@
 +import sys
 +
 +    def __init__(self, space, codec, errors):
 +        self.errors = errors
 +        self.codec = codec
++        self.state = c_codecs.pypy_cjk_dec_init_state(codec.codec)
 +
 +        # not exposed
 +        self.pendingsize = 0
 +        self.pending = None # XXX:
 +
++    def __del__(self):
++        if self.state:
++            c_codecs.pypy_cjk_dec_free_state(self.state)
++
 +    @unwrap_spec(errors="str_or_None")
 +    def mbidecoder_new(space, w_subtype, errors='strict'):
 +        codec = space.getattr(w_subtype, space.wrap('codec'))
 +
 +    @unwrap_spec(final=int)
 +    def decode(self, space, w_input, final=0):
-+        # get a butter from input
-+        #input = space.unicode_w(w_input)
-+        
-+        # check pending size
-+        # prep buf
-+        # feed buf
-+        # deal w/ pending
++        input = space.str_w(w_input)
 +        inputlen = len(input)
 +        origpending = w_ctx.pendingsize
 +        if origpending > 0:
 +            inputlen += origpending
 +            w_ctx.pendingsize = 0
 +
-+        pypy_cjk_dec_prepare_buffer(self.decodebuf, inbuf, inputlen)
++        codec = self.codec.codec
++        decodebuf = c_codecs.pypy_cjk_dec_init(codec, input, inputlen)
 +
 +        state = space.fromcache(CodecState)
 +        try:
  @unwrap_spec(name=str)
  def getcodec(space, name):
      try:
-diff -r b5bc2f812d76 pypy/module/_multibytecodec/test/test_app_codecs.py
---- a/pypy/module/_multibytecodec/test/test_app_codecs.py	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/module/_multibytecodec/test/test_app_codecs.py	Tue Jul 26 10:14:42 2011 -0700
-@@ -106,3 +106,67 @@
+diff -r 54515dec1d20 pypy/module/_multibytecodec/test/test_app_codecs.py
+--- a/pypy/module/_multibytecodec/test/test_app_codecs.py	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/module/_multibytecodec/test/test_app_codecs.py	Fri Jul 29 16:28:46 2011 -0700
+@@ -106,3 +106,75 @@
          repl = u"\u2014"
          s = u"\uDDA1".encode("gbk", "test.multi_bad_handler")
          assert s == '\xA1\xAA'
 +        raises(UnicodeEncodeError, encoder.encode, u'\u0123')
 +        s = encoder.encode(u'', True)
 +        assert s == '\xa9\xdc'
-\ No newline at end of file
-diff -r b5bc2f812d76 pypy/translator/c/src/cjkcodecs/multibytecodec.c
---- a/pypy/translator/c/src/cjkcodecs/multibytecodec.c	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.c	Tue Jul 26 10:14:42 2011 -0700
-@@ -169,9 +169,8 @@
++
++    def test_issue5640(self):
++        import codecs
++        encoder = codecs.getincrementalencoder('shift-jis')('backslashreplace')
++        s = encoder.encode(u'\xff')
++        assert s ==  b'\\xff'
++        s = encoder.encode(u'\n')
++        assert s == b'\n'
+diff -r 54515dec1d20 pypy/translator/c/src/cjkcodecs/multibytecodec.c
+--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.c	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.c	Fri Jul 29 16:28:46 2011 -0700
+@@ -30,6 +30,50 @@
+   return NULL;
+ }
+ 
++MultibyteCodec_State *pypy_cjk_dec_init_state(const MultibyteCodec *codec)
++{
++  MultibyteCodec_State *state = malloc(sizeof(MultibyteCodec_State));
++  if (!state)
++    return NULL;
++  if (codec->decinit != NULL && codec->decinit(state, codec->config) != 0) {
++    free(state);
++    return NULL;
++  }
++  return state;
++}
++
++void pypy_cjk_dec_free_state(MultibyteCodec_State *state)
++{
++  free(state);
++}
++
++struct pypy_cjk_dec_s *pypy_cjk_prepare_buffer(const MultibyteCodec *codec,
++                                               char *inbuf, Py_ssize_t inlen)
++{
++  struct pypy_cjk_dec_s *d = malloc(sizeof(struct pypy_cjk_dec_s));
++  if (!d)
++    return NULL;
++  if (codec->decinit != NULL && codec->decinit(&d->state, codec->config) != 0)
++    goto errorexit;
++
++  d->codec = codec;
++  d->inbuf_start = inbuf;
++  d->inbuf = inbuf;
++  d->inbuf_end = inbuf + inlen;
++  d->outbuf_start = (inlen <= (PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) ?
++                     malloc(inlen * sizeof(Py_UNICODE)) :
++                     NULL);
++  if (!d->outbuf_start)
++    goto errorexit;
++  d->outbuf = d->outbuf_start;
++  d->outbuf_end = d->outbuf_start + inlen;
++  return d;
++
++ errorexit:
++  free(d);
++  return NULL;
++}
++
+ void pypy_cjk_dec_free(struct pypy_cjk_dec_s *d)
+ {
+   free(d->outbuf_start);
+@@ -169,9 +213,8 @@
  
  #define MBENC_RESET     MBENC_MAX<<1
  
    while (1)
      {
        Py_ssize_t r;
-diff -r b5bc2f812d76 pypy/translator/c/src/cjkcodecs/multibytecodec.h
---- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h	Sun Jul 24 17:13:53 2011 -0700
-+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h	Tue Jul 26 10:14:42 2011 -0700
+diff -r 54515dec1d20 pypy/translator/c/src/cjkcodecs/multibytecodec.h
+--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h	Fri Jul 29 14:17:43 2011 -0500
++++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h	Fri Jul 29 16:28:46 2011 -0700
 @@ -115,7 +115,7 @@
  struct pypy_cjk_enc_s *pypy_cjk_enc_init(const MultibyteCodec *codec,
                                           Py_UNICODE *inbuf, Py_ssize_t inlen);
+5063903bba091be7601ff86f1e8c913251d037b9:multibytecodec