Commits

Grant Eagon  committed db085b7

Fix an issue with unicode errors causing files to fail download. Stole a bit of code from Django to do the unicode string conversion. Admittedly, there's probably more code in the django_encoding.py file than is necessary, but leaving it for now in case we run into other requirements.

  • Participants
  • Parent commits 3c60cbe

Comments (0)

Files changed (2)

 import sublime, sublime_plugin
 import json, os, threading, re
 import urllib, base64, urllib2
+from django_encoding import smart_str
 
 from datetime import tzinfo, timedelta, datetime
 
         if ('attachment' in asset.keys()):
             value = base64.b64decode(asset['attachment'])
         else:
-            value = asset['value']
+            value = smart_str(asset['value'])
 
         if not os.path.exists(os.path.dirname(asset_full_name)):
             os.makedirs(os.path.dirname(asset_full_name))

File django_encoding.py

+# This file was ripped from django 1.4 with some minor mods
+## Django stuff
+import types
+import urllib
+import locale
+import datetime
+import codecs
+from decimal import Decimal
+
+# pulled this in from django.utils.functional
+class Promise(object):
+    """
+    This is just a base class for the proxy class created in
+    the closure of the lazy function. It can be used to recognize
+    promises in code.
+    """
+    pass
+
+class DjangoUnicodeDecodeError(UnicodeDecodeError):
+    def __init__(self, obj, *args):
+        self.obj = obj
+        UnicodeDecodeError.__init__(self, *args)
+
+    def __str__(self):
+        original = UnicodeDecodeError.__str__(self)
+        return '%s. You passed in %r (%s)' % (original, self.obj,
+                type(self.obj))
+
+class StrAndUnicode(object):
+    """
+    A class whose __str__ returns its __unicode__ as a UTF-8 bytestring.
+
+    Useful as a mix-in.
+    """
+    def __str__(self):
+        return self.__unicode__().encode('utf-8')
+
+def smart_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
+    """
+    Returns a unicode object representing 's'. Treats bytestrings using the
+    'encoding' codec.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
+    """
+    if isinstance(s, Promise):
+        # The input is the result of a gettext_lazy() call.
+        return s
+    return force_unicode(s, encoding, strings_only, errors)
+
+def is_protected_type(obj):
+    """Determine if the object instance is of a protected type.
+
+    Objects of protected types are preserved as-is when passed to
+    force_unicode(strings_only=True).
+    """
+    return isinstance(obj, (
+        types.NoneType,
+        int, long,
+        datetime.datetime, datetime.date, datetime.time,
+        float, Decimal)
+    )
+
+def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'):
+    """
+    Similar to smart_unicode, except that lazy instances are resolved to
+    strings, rather than kept as lazy objects.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
+    """
+    # Handle the common case first, saves 30-40% in performance when s
+    # is an instance of unicode. This function gets called often in that
+    # setting.
+    if isinstance(s, unicode):
+        return s
+    if strings_only and is_protected_type(s):
+        return s
+    try:
+        if not isinstance(s, basestring,):
+            if hasattr(s, '__unicode__'):
+                s = unicode(s)
+            else:
+                try:
+                    s = unicode(str(s), encoding, errors)
+                except UnicodeEncodeError:
+                    if not isinstance(s, Exception):
+                        raise
+                    # If we get to here, the caller has passed in an Exception
+                    # subclass populated with non-ASCII data without special
+                    # handling to display as a string. We need to handle this
+                    # without raising a further exception. We do an
+                    # approximation to what the Exception's standard str()
+                    # output should be.
+                    s = u' '.join([force_unicode(arg, encoding, strings_only,
+                            errors) for arg in s])
+        elif not isinstance(s, unicode):
+            # Note: We use .decode() here, instead of unicode(s, encoding,
+            # errors), so that if s is a SafeString, it ends up being a
+            # SafeUnicode at the end.
+            s = s.decode(encoding, errors)
+    except UnicodeDecodeError as e:
+        if not isinstance(s, Exception):
+            raise DjangoUnicodeDecodeError(s, *e.args)
+        else:
+            # If we get to here, the caller has passed in an Exception
+            # subclass populated with non-ASCII bytestring data without a
+            # working unicode method. Try to handle this without raising a
+            # further exception by individually forcing the exception args
+            # to unicode.
+            s = u' '.join([force_unicode(arg, encoding, strings_only,
+                    errors) for arg in s])
+    return s
+
+def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'):
+    """
+    Returns a bytestring version of 's', encoded as specified in 'encoding'.
+
+    If strings_only is True, don't convert (some) non-string-like objects.
+    """
+    if strings_only and isinstance(s, (types.NoneType, int)):
+        return s
+    if isinstance(s, Promise):
+        return unicode(s).encode(encoding, errors)
+    elif not isinstance(s, basestring):
+        try:
+            return str(s)
+        except UnicodeEncodeError:
+            if isinstance(s, Exception):
+                # An Exception subclass containing non-ASCII data that doesn't
+                # know how to print itself properly. We shouldn't raise a
+                # further exception.
+                return ' '.join([smart_str(arg, encoding, strings_only,
+                        errors) for arg in s])
+            return unicode(s).encode(encoding, errors)
+    elif isinstance(s, unicode):
+        return s.encode(encoding, errors)
+    elif s and encoding != 'utf-8':
+        return s.decode('utf-8', errors).encode(encoding, errors)
+    else:
+        return s
+
+def filepath_to_uri(path):
+    """Convert an file system path to a URI portion that is suitable for
+    inclusion in a URL.
+
+    We are assuming input is either UTF-8 or unicode already.
+
+    This method will encode certain chars that would normally be recognized as
+    special chars for URIs.  Note that this method does not encode the '
+    character, as it is a valid character within URIs.  See
+    encodeURIComponent() JavaScript function for more details.
+
+    Returns an ASCII string containing the encoded result.
+    """
+    if path is None:
+        return path
+    # I know about `os.sep` and `os.altsep` but I want to leave
+    # some flexibility for hardcoding separators.
+    return urllib.quote(smart_str(path).replace("\\", "/"), safe="/~!*()'")
+
+# The encoding of the default system locale but falls back to the
+# given fallback encoding if the encoding is unsupported by python or could
+# not be determined.  See tickets #10335 and #5846
+try:
+    DEFAULT_LOCALE_ENCODING = locale.getdefaultlocale()[1] or 'ascii'
+    codecs.lookup(DEFAULT_LOCALE_ENCODING)
+except:
+    DEFAULT_LOCALE_ENCODING = 'ascii'
+
+## End django stuff