Georg Brandl avatar Georg Brandl committed 47a43f2

#71: If a decoding error occurs in source files, print a warning and replace the characters by "?".

Comments (0)

Files changed (2)

 Release 0.5.2 (in development)
 ==============================
 
+* #71: If a decoding error occurs in source files, print a
+  warning and replace the characters by "?".
+
 * Fix a problem in the HTML search if the index takes too long
   to load.
 

sphinx/environment.py

 import time
 import heapq
 import types
+import codecs
 import imghdr
 import difflib
 import cPickle as pickle
 
     # --------- SINGLE FILE READING --------------------------------------------
 
+    def warn_and_replace(self, error):
+        """
+        Custom decoding error handler that warns and replaces.
+        """
+        linestart = error.object.rfind('\n', None, error.start)
+        lineend = error.object.find('\n', error.start)
+        if lineend == -1: lineend = len(error.object)
+        lineno = error.object.count('\n', 0, error.start) + 1
+        self.warn(self.docname, 'undecodable source characters, '
+                  'replacing with "?": %r' %
+                  (error.object[linestart+1:error.start] + '>>>' +
+                   error.object[error.start:error.end] + '<<<' +
+                   error.object[error.end:lineend]), lineno)
+        return (u'?', error.end)
+
     def read_doc(self, docname, src_path=None, save_parsed=True, app=None):
         """
         Parse a file and add/update inventory entries for the doctree.
         self.docname = docname
         self.settings['input_encoding'] = self.config.source_encoding
 
+        codecs.register_error('sphinx', self.warn_and_replace)
+
         class SphinxSourceClass(FileInput):
+            def decode(self_, data):
+                return data.decode(self_.encoding, 'sphinx')
+
             def read(self):
                 data = FileInput.read(self)
                 if app:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.