Commits

Philip Jenvey committed 1c77bd0

add a 'locale' codec for use by fsdecode/encode during interpreter bootstrap,
works via POSIX wcstombs/mbrtowc (and a chunk of C from cpython)

  • Participants
  • Parent commits 7939f83
  • Branches py3k

Comments (0)

Files changed (3)

pypy/module/_codecs/locale.c

+/* From CPython 3.2.3's fileutils.c, and _Py_normalize_encoding from
+   unicodeobject.c
+*/
+/*
+#include "Python.h"
+*/
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+#define PyMem_Malloc malloc
+#define PyMem_Free free
+/* C99 but recent Windows has it */
+#define HAVE_MBRTOWC 1
+
+#ifdef MS_WINDOWS
+#  include <windows.h>
+#endif
+
+#ifdef HAVE_LANGINFO_H
+#include <locale.h>
+#include <langinfo.h>
+#endif
+
+#if 0 && defined(__APPLE__)
+extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size);
+#endif
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+#if 0
+extern int _pypy_normalize_encoding(const char *, char *, size_t);
+#endif
+
+/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale.
+   On these operating systems, nl_langinfo(CODESET) announces an alias of the
+   ASCII encoding, whereas mbstowcs() and wcstombs() functions use the
+   ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use
+   locale.getpreferredencoding() codec. For example, if command line arguments
+   are decoded by mbstowcs() and encoded back by os.fsencode(), we get a
+   UnicodeEncodeError instead of retrieving the original byte string.
+
+   The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",
+   nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least
+   one byte in range 0x80-0xff can be decoded from the locale encoding. The
+   workaround is also enabled on error, for example if getting the locale
+   failed.
+
+   Values of locale_is_ascii:
+
+       1: the workaround is used: _Py_wchar2char() uses
+          encode_ascii_surrogateescape() and _Py_char2wchar() uses
+          decode_ascii_surrogateescape()
+       0: the workaround is not used: _Py_wchar2char() uses wcstombs() and
+          _Py_char2wchar() uses mbstowcs()
+      -1: unknown, need to call check_force_ascii() to get the value
+*/
+static int force_ascii = -1;
+
+static int
+_pypy_check_force_ascii(void)
+{
+    char *loc;
+#if defined(HAVE_LANGINFO_H) && defined(CODESET)
+    char *codeset, **alias;
+    char encoding[100];
+    int is_ascii;
+    unsigned int i;
+    char* ascii_aliases[] = {
+        "ascii",
+        "646",
+        "ansi-x3.4-1968",
+        "ansi-x3-4-1968",
+        "ansi-x3.4-1986",
+        "cp367",
+        "csascii",
+        "ibm367",
+        "iso646-us",
+        "iso-646.irv-1991",
+        "iso-ir-6",
+        "us",
+        "us-ascii",
+        NULL
+    };
+#endif
+
+    loc = setlocale(LC_CTYPE, NULL);
+    if (loc == NULL)
+        goto error;
+    if (strcmp(loc, "C") != 0) {
+        /* the LC_CTYPE locale is different than C */
+        return 0;
+    }
+
+#if defined(HAVE_LANGINFO_H) && defined(CODESET)
+    codeset = nl_langinfo(CODESET);
+    if (!codeset || codeset[0] == '\0') {
+        /* CODESET is not set or empty */
+        goto error;
+    }
+    if (!_pypy_normalize_encoding(codeset, encoding, sizeof(encoding)))
+        goto error;
+
+    is_ascii = 0;
+    for (alias=ascii_aliases; *alias != NULL; alias++) {
+        if (strcmp(encoding, *alias) == 0) {
+            is_ascii = 1;
+            break;
+        }
+    }
+    if (!is_ascii) {
+        /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */
+        return 0;
+    }
+
+    for (i=0x80; i<0xff; i++) {
+        unsigned char ch;
+        wchar_t wch;
+        size_t res;
+
+        ch = (unsigned char)i;
+        res = mbstowcs(&wch, (char*)&ch, 1);
+        if (res != (size_t)-1) {
+            /* decoding a non-ASCII character from the locale encoding succeed:
+               the locale encoding is not ASCII, force ASCII */
+            return 1;
+        }
+    }
+    /* None of the bytes in the range 0x80-0xff can be decoded from the locale
+       encoding: the locale encoding is really ASCII */
+    return 0;
+#else
+    /* nl_langinfo(CODESET) is not available: always force ASCII */
+    return 1;
+#endif
+
+error:
+    /* if an error occured, force the ASCII encoding */
+    return 1;
+}
+
+static char*
+_pypy_encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos)
+{
+    char *result = NULL, *out;
+    size_t len, i;
+    wchar_t ch;
+
+    if (error_pos != NULL)
+        *error_pos = (size_t)-1;
+
+    len = wcslen(text);
+
+    result = PyMem_Malloc(len + 1);  /* +1 for NUL byte */
+    if (result == NULL)
+        return NULL;
+
+    out = result;
+    for (i=0; i<len; i++) {
+        ch = text[i];
+
+        if (ch <= 0x7f) {
+            /* ASCII character */
+            *out++ = (char)ch;
+        }
+        else if (0xdc80 <= ch && ch <= 0xdcff) {
+            /* UTF-8b surrogate */
+            *out++ = (char)(ch - 0xdc00);
+        }
+        else {
+            if (error_pos != NULL)
+                *error_pos = i;
+            PyMem_Free(result);
+            return NULL;
+        }
+    }
+    *out = '\0';
+    return result;
+}
+#endif   /* !defined(__APPLE__) && !defined(MS_WINDOWS) */
+
+#if !defined(__APPLE__) && (!defined(MS_WINDOWS) || !defined(HAVE_MBRTOWC))
+static wchar_t*
+_pypy_decode_ascii_surrogateescape(const char *arg, size_t *size)
+{
+    wchar_t *res;
+    unsigned char *in;
+    wchar_t *out;
+
+    res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+    if (!res)
+        return NULL;
+
+    in = (unsigned char*)arg;
+    out = res;
+    while(*in)
+        if(*in < 128)
+            *out++ = *in++;
+        else
+            *out++ = 0xdc00 + *in++;
+    *out = 0;
+    if (size != NULL)
+        *size = out - res;
+    return res;
+}
+#endif
+
+
+/* Decode a byte string from the locale encoding with the
+   surrogateescape error handler (undecodable bytes are decoded as characters
+   in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
+   character, escape the bytes using the surrogateescape error handler instead
+   of decoding them.
+
+   Use _Py_wchar2char() to encode the character string back to a byte string.
+
+   Return a pointer to a newly allocated wide character string (use
+   PyMem_Free() to free the memory) and write the number of written wide
+   characters excluding the null character into *size if size is not NULL, or
+   NULL on error (conversion or memory allocation error).
+
+   Conversion errors should never happen, unless there is a bug in the C
+   library. */
+wchar_t*
+pypy_char2wchar(const char* arg, size_t *size)
+{
+#if 0 && defined(__APPLE__)
+    wchar_t *wstr;
+    wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg));
+    if (size != NULL) {
+        if (wstr != NULL)
+            *size = wcslen(wstr);
+        else
+            *size = (size_t)-1;
+    }
+    return wstr;
+#else
+    wchar_t *res;
+    size_t argsize;
+    size_t count;
+    unsigned char *in;
+    wchar_t *out;
+#ifdef HAVE_MBRTOWC
+    mbstate_t mbs;
+#endif
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+/*#ifndef MS_WINDOWS*/
+    if (force_ascii == -1)
+        force_ascii = _pypy_check_force_ascii();
+
+    if (force_ascii) {
+        /* force ASCII encoding to workaround mbstowcs() issue */
+        res = _pypy_decode_ascii_surrogateescape(arg, size);
+        if (res == NULL)
+            goto oom;
+        return res;
+    }
+#endif
+
+#ifdef HAVE_BROKEN_MBSTOWCS
+    /* Some platforms have a broken implementation of
+     * mbstowcs which does not count the characters that
+     * would result from conversion.  Use an upper bound.
+     */
+    argsize = strlen(arg);
+#else
+    argsize = mbstowcs(NULL, arg, 0);
+#endif
+    if (argsize != (size_t)-1) {
+        res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+        if (!res)
+            goto oom;
+        count = mbstowcs(res, arg, argsize+1);
+        if (count != (size_t)-1) {
+            wchar_t *tmp;
+            /* Only use the result if it contains no
+               surrogate characters. */
+            for (tmp = res; *tmp != 0 &&
+                         (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+                ;
+            if (*tmp == 0) {
+                if (size != NULL)
+                    *size = count;
+                return res;
+            }
+        }
+        PyMem_Free(res);
+    }
+    /* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+    /* Overallocate; as multi-byte characters are in the argument, the
+       actual output could use less memory. */
+    argsize = strlen(arg) + 1;
+    res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+    if (!res)
+        goto oom;
+    in = (unsigned char*)arg;
+    out = res;
+    memset(&mbs, 0, sizeof mbs);
+    while (argsize) {
+        size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+        if (converted == 0)
+            /* Reached end of string; null char stored. */
+            break;
+        if (converted == (size_t)-2) {
+            /* Incomplete character. This should never happen,
+               since we provide everything that we have -
+               unless there is a bug in the C library, or I
+               misunderstood how mbrtowc works. */
+            fprintf(stderr, "unexpected mbrtowc result -2\n");
+            PyMem_Free(res);
+            return NULL;
+        }
+        if (converted == (size_t)-1) {
+            /* Conversion error. Escape as UTF-8b, and start over
+               in the initial shift state. */
+            *out++ = 0xdc00 + *in++;
+            argsize--;
+            memset(&mbs, 0, sizeof mbs);
+            continue;
+        }
+        if (*out >= 0xd800 && *out <= 0xdfff) {
+            /* Surrogate character.  Escape the original
+               byte sequence with surrogateescape. */
+            argsize -= converted;
+            while (converted--)
+                *out++ = 0xdc00 + *in++;
+            continue;
+        }
+        /* successfully converted some bytes */
+        in += converted;
+        argsize -= converted;
+        out++;
+    }
+    if (size != NULL)
+        *size = out - res;
+#else   /* HAVE_MBRTOWC */
+    /* Cannot use C locale for escaping; manually escape as if charset
+       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+       correctly in the locale's charset, which must be an ASCII superset. */
+    res = _pypy_decode_ascii_surrogateescape(arg, size);
+    if (res == NULL)
+        goto oom;
+#endif   /* HAVE_MBRTOWC */
+    return res;
+oom:
+    fprintf(stderr, "out of memory\n");
+    return NULL;
+#endif   /* __APPLE__ */
+}
+
+/* Encode a (wide) character string to the locale encoding with the
+   surrogateescape error handler (characters in range U+DC80..U+DCFF are
+   converted to bytes 0x80..0xFF).
+
+   This function is the reverse of _Py_char2wchar().
+
+   Return a pointer to a newly allocated byte string (use PyMem_Free() to free
+   the memory), or NULL on conversion or memory allocation error.
+
+   If error_pos is not NULL: *error_pos is the index of the invalid character
+   on conversion error, or (size_t)-1 otherwise. */
+char*
+pypy_wchar2char(const wchar_t *text, size_t *error_pos)
+{
+#if 0 && defined(__APPLE__)
+    Py_ssize_t len;
+    PyObject *unicode, *bytes = NULL;
+    char *cpath;
+
+    unicode = PyUnicode_FromWideChar(text, wcslen(text));
+    if (unicode == NULL)
+        return NULL;
+
+    bytes = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
+                                 PyUnicode_GET_SIZE(unicode),
+                                 "surrogateescape");
+    Py_DECREF(unicode);
+    if (bytes == NULL) {
+        PyErr_Clear();
+        if (error_pos != NULL)
+            *error_pos = (size_t)-1;
+        return NULL;
+    }
+
+    len = PyBytes_GET_SIZE(bytes);
+    cpath = PyMem_Malloc(len+1);
+    if (cpath == NULL) {
+        PyErr_Clear();
+        Py_DECREF(bytes);
+        if (error_pos != NULL)
+            *error_pos = (size_t)-1;
+        return NULL;
+    }
+    memcpy(cpath, PyBytes_AsString(bytes), len + 1);
+    Py_DECREF(bytes);
+    return cpath;
+#else   /* __APPLE__ */
+    const size_t len = wcslen(text);
+    char *result = NULL, *bytes = NULL;
+    size_t i, size, converted;
+    wchar_t c, buf[2];
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+/*#ifndef MS_WINDOWS*/
+    if (force_ascii == -1)
+        force_ascii = _pypy_check_force_ascii();
+
+    if (force_ascii)
+        return _pypy_encode_ascii_surrogateescape(text, error_pos);
+#endif
+
+    /* The function works in two steps:
+       1. compute the length of the output buffer in bytes (size)
+       2. outputs the bytes */
+    size = 0;
+    buf[1] = 0;
+    while (1) {
+        for (i=0; i < len; i++) {
+            c = text[i];
+            if (c >= 0xdc80 && c <= 0xdcff) {
+                /* UTF-8b surrogate */
+                if (bytes != NULL) {
+                    *bytes++ = c - 0xdc00;
+                    size--;
+                }
+                else
+                    size++;
+                continue;
+            }
+            else {
+                buf[0] = c;
+                if (bytes != NULL)
+                    converted = wcstombs(bytes, buf, size);
+                else
+                    converted = wcstombs(NULL, buf, 0);
+                if (converted == (size_t)-1) {
+                    if (result != NULL)
+                        PyMem_Free(result);
+                    if (error_pos != NULL)
+                        *error_pos = i;
+                    return NULL;
+                }
+                if (bytes != NULL) {
+                    bytes += converted;
+                    size -= converted;
+                }
+                else
+                    size += converted;
+            }
+        }
+        if (result != NULL) {
+            *bytes = '\0';
+            break;
+        }
+
+        size += 1; /* nul byte at the end */
+        result = PyMem_Malloc(size);
+        if (result == NULL) {
+            if (error_pos != NULL)
+                *error_pos = (size_t)-1;
+            return NULL;
+        }
+        bytes = result;
+    }
+    return result;
+#endif   /* __APPLE__ */
+}
+
+void
+pypy_char2wchar_free(wchar_t *text)
+{
+    PyMem_Free(text);
+}
+
+void
+pypy_wchar2char_free(char *bytes)
+{
+    PyMem_Free(bytes);
+}
+
+#define Py_ISUPPER isupper
+#define Py_TOLOWER tolower
+
+/* Convert encoding to lower case and replace '_' with '-' in order to
+   catch e.g. UTF_8. Return 0 on error (encoding is longer than lower_len-1),
+   1 on success. */
+int
+_pypy_normalize_encoding(const char *encoding,
+                   char *lower,
+                   size_t lower_len)
+{
+    const char *e;
+    char *l;
+    char *l_end;
+
+    e = encoding;
+    l = lower;
+    l_end = &lower[lower_len - 1];
+    while (*e) {
+        if (l == l_end)
+            return 0;
+        if (Py_ISUPPER(*e)) {
+            *l++ = Py_TOLOWER(*e++);
+        }
+        else if (*e == '_') {
+            *l++ = '-';
+            e++;
+        }
+        else {
+            *l++ = *e++;
+        }
+    }
+    *l = '\0';
+    return 1;
+}

pypy/module/_codecs/locale.py

+"""
+Provides internal 'locale' codecs (via POSIX wcstombs/mbrtowc) for use
+by PyUnicode_Decode/EncodeFSDefault during interpreter bootstrap
+"""
+import os
+import py
+import sys
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rstring import UnicodeBuilder, assert_str0
+from rpython.rlib.runicode import (code_to_unichr,
+    default_unicode_error_decode, default_unicode_error_encode)
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+if we_are_translated():
+    UNICHAR_SIZE = rffi.sizeof(lltype.UniChar)
+else:
+    UNICHAR_SIZE = 2 if sys.maxunicode == 0xFFFF else 4
+MERGE_SURROGATES = UNICHAR_SIZE == 2 and rffi.sizeof(rffi.WCHAR_T) == 4
+
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+    separate_module_files=[cwd.join('locale.c')],
+    export_symbols=['pypy_char2wchar', 'pypy_char2wchar_free',
+                    'pypy_wchar2char', 'pypy_wchar2char_free'])
+
+def llexternal(*args, **kwargs):
+    kwargs.setdefault('compilation_info', eci)
+    kwargs.setdefault('sandboxsafe', True)
+    kwargs.setdefault('_nowrapper', True)
+    return rffi.llexternal(*args, **kwargs)
+
+# An actual wchar_t*, rffi.CWCHARP is an array of UniChar (possibly on a
+# narrow build)
+RAW_WCHARP = lltype.Ptr(lltype.Array(rffi.WCHAR_T, hints={'nolength': True}))
+pypy_char2wchar = llexternal('pypy_char2wchar', [rffi.CCHARP, rffi.SIZE_TP],
+                             RAW_WCHARP)
+pypy_char2wchar_free = llexternal('pypy_char2wchar_free', [RAW_WCHARP],
+                                  lltype.Void)
+pypy_wchar2char = llexternal('pypy_wchar2char', [RAW_WCHARP, rffi.SIZE_TP],
+                             rffi.CCHARP)
+pypy_wchar2char_free = llexternal('pypy_wchar2char_free', [rffi.CCHARP],
+                                  lltype.Void)
+
+
+def unicode_encode_locale_surrogateescape(u, errorhandler=None):
+    """Encode unicode via the locale codecs (POSIX wcstombs) with the
+    surrogateescape handler.
+
+    The optional errorhandler is only called in the case of fatal
+    errors.
+    """
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+
+    with lltype.scoped_alloc(rffi.SIZE_TP.TO, 1) as errorposp:
+        with scoped_unicode2rawwcharp(u) as ubuf:
+            sbuf = pypy_wchar2char(ubuf, errorposp)
+        try:
+            if sbuf is None:
+                errorpos = rffi.cast(lltype.Signed, errorposp[0])
+                if errorpos == -1:
+                    raise MemoryError
+                errmsg = _errmsg("pypy_wchar2char")
+                errorhandler('strict', 'filesystemencoding', errmsg, u,
+                             errorpos, errorpos + 1)
+            return rffi.charp2str(sbuf)
+        finally:
+            pypy_wchar2char_free(sbuf)
+
+
+def unicode_decode_locale_surrogateescape(s, errorhandler=None):
+    """Decode strs via the locale codecs (POSIX mrbtowc) with the
+    surrogateescape handler.
+
+    The optional errorhandler is only called in the case of fatal
+    errors.
+    """
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+
+    with lltype.scoped_alloc(rffi.SIZE_TP.TO, 1) as sizep:
+        with rffi.scoped_str2charp(s) as sbuf:
+            ubuf = pypy_char2wchar(sbuf, sizep)
+        try:
+            if ubuf is None:
+                errmsg = _errmsg("pypy_char2wchar")
+                errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
+            size = rffi.cast(lltype.Signed, sizep[0])
+            return rawwcharp2unicoden(ubuf, size)
+        finally:
+            pypy_char2wchar_free(ubuf)
+
+
+def _errmsg(what):
+    from rpython.rlib import rposix
+    errmsg = os.strerror(rposix.get_errno())
+    return "%s failed" % what if errmsg is None else errmsg
+
+
+class scoped_unicode2rawwcharp:
+    def __init__(self, value):
+        if value is not None:
+            self.buf = unicode2rawwcharp(value)
+        else:
+            self.buf = lltype.nullptr(RAW_WCHARP.TO)
+    def __enter__(self):
+        return self.buf
+    def __exit__(self, *args):
+        if self.buf:
+            lltype.free(self.buf, flavor='raw')
+
+def unicode2rawwcharp(u):
+    """unicode -> raw wchar_t*"""
+    size = _unicode2cwcharp_loop(u, None) if MERGE_SURROGATES else len(u)
+    array = lltype.malloc(RAW_WCHARP.TO, size + 1, flavor='raw')
+    array[size] = rffi.cast(rffi.WCHAR_T, u'\x00')
+    _unicode2cwcharp_loop(u, array)
+    return array
+unicode2rawwcharp._annenforceargs_ = [unicode]
+
+def _unicode2cwcharp_loop(u, array):
+    write = array is not None
+    ulen = len(u)
+    count = i = 0
+    while i < ulen:
+        oc = ord(u[i])
+        if (MERGE_SURROGATES and
+            0xD800 <= oc <= 0xDBFF and i + 1 < ulen and
+            0xDC00 <= ord(u[i + 1]) <= 0xDFFF):
+            if write:
+                merged = (((oc & 0x03FF) << 10) |
+                          (ord(u[i + 1]) & 0x03FF)) + 0x10000
+                array[count] = rffi.cast(rffi.WCHAR_T, merged)
+            i += 2
+        else:
+            if write:
+                array[count] = rffi.cast(rffi.WCHAR_T, oc)
+            i += 1
+        count += 1
+    return count
+unicode2rawwcharp._annenforceargs_ = [unicode, None]
+
+
+def rawwcharp2unicoden(wcp, maxlen):
+    b = UnicodeBuilder(maxlen)
+    i = 0
+    while i < maxlen and wcp[i] != u'\x00':
+        b.append(code_to_unichr(wcp[i]))
+        i += 1
+    return assert_str0(b.build())
+unicode2rawwcharp._annenforceargs_ = [None, int]

pypy/module/_codecs/test/test_locale.py

+# coding: utf-8
+import py
+from pypy.module._codecs import interp_codecs
+from pypy.module._codecs.locale import (
+    unicode_decode_locale_surrogateescape,
+    unicode_encode_locale_surrogateescape)
+from rpython.rlib import rlocale, runicode
+
+class TestLocaleCodec(object):
+
+    def setup_class(cls):
+        from rpython.rlib import rlocale
+        cls.oldlocale = rlocale.setlocale(rlocale.LC_ALL, None)
+
+    def teardown_class(cls):
+        if hasattr(cls, 'oldlocale'):
+            from rpython.rlib import rlocale
+            rlocale.setlocale(rlocale.LC_ALL, cls.oldlocale)
+
+    def getdecoder(self, encoding):
+        return getattr(runicode, "str_decode_%s" % encoding.replace("-", "_"))
+
+    def getencoder(self, encoding):
+        return getattr(runicode,
+                       "unicode_encode_%s" % encoding.replace("-", "_"))
+
+    def getstate(self):
+        return self.space.fromcache(interp_codecs.CodecState)
+
+    def setlocale(self, locale):
+        from rpython.rlib import rlocale
+        try:
+            rlocale.setlocale(rlocale.LC_ALL, locale)
+        except rlocale.LocaleError:
+            py.test.skip("%s locale unsupported" % locale)
+
+    def test_encode_locale(self):
+        self.setlocale("en_US.UTF-8")
+        locale_encoder = unicode_encode_locale_surrogateescape
+        utf8_encoder = self.getencoder('utf-8')
+        for val in u'foo', u' 日本', u'\U0001320C':
+            assert (locale_encoder(val) ==
+                    utf8_encoder(val, len(val), None))
+
+    def test_encode_locale_errorhandler(self):
+        self.setlocale("en_US.UTF-8")
+        locale_encoder = unicode_encode_locale_surrogateescape
+        utf8_encoder = self.getencoder('utf-8')
+        encode_error_handler = self.getstate().encode_error_handler
+        for val in u'foo\udc80bar', u'\udcff\U0001320C':
+            expected = utf8_encoder(val, len(val), 'surrogateescape',
+                                    encode_error_handler)
+            assert locale_encoder(val) == expected
+
+    def test_decode_locale(self):
+        self.setlocale("en_US.UTF-8")
+        locale_decoder = unicode_decode_locale_surrogateescape
+        utf8_decoder = self.getdecoder('utf-8')
+        for val in 'foo', ' \xe6\x97\xa5\xe6\x9c\xac', '\xf0\x93\x88\x8c':
+            assert (locale_decoder(val) ==
+                    utf8_decoder(val, len(val), None)[0])
+
+    def test_decode_locale_errorhandler(self):
+        self.setlocale("en_US.UTF-8")
+        locale_decoder = unicode_decode_locale_surrogateescape
+        utf8_decoder = self.getdecoder('utf-8')
+        decode_error_handler = self.getstate().decode_error_handler
+        val = 'foo\xe3bar'
+        expected = utf8_decoder(val, len(val), 'surrogateescape', True,
+                                decode_error_handler)[0]
+        assert locale_decoder(val) == expected