Lynn Rees avatar Lynn Rees committed 64f98a9

- normalize

Comments (0)

Files changed (2)

 from functools import partial
 from keyword import iskeyword
 from operator import itemgetter
+from unicodedata import normalize
 from importlib import import_module
 from collections import Sequence, Mapping
 
 maporseq = isfactory((Mapping, Sequence))
 ismapping = isfactory(Mapping)
 issequence = isfactory(Sequence)
+norm = partial(normalize, 'NFKD')
 # illegal characters for Python names
 ic = frozenset('()[]{}@,:`=;+*/%&|^><\'"#\\$?!~'.split())
 
 from importlib import import_module
 from operator import attrgetter, methodcaller, lt, gt
 
-from stuf.base import isfactory
+from stuf.base import isfactory, norm
 
 intern = backport('__builtin__.intern', 'sys.intern')
 OrderedDict = backport('collections.OrderedDict', 'ordereddict.OrderedDict')
 def tounicode(thing, encoding='utf-8', errors='strict'):
     '''Convert string `thing` to unicode string with `encoding`.'''
     if isbytes(thing):
-        return thing.decode(encoding, errors)
+        return norm(thing).decode(encoding, errors)
     return utfme(utfme(thing).encode(encoding, errors), encoding, errors)
 
 
 def tobytes(thing, encoding='utf-8', errors='strict'):
     '''Convert string `thing` to byte string `encoding`.'''
-    return thing if isbytes(thing) else utfme(thing).encode(encoding, errors)
+    return thing if isbytes(thing) else utfme(norm(thing)).encode(encoding, errors)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.