Harald Klimach avatar Harald Klimach committed 72c1a5d

Do not try to tokenize non-python files when using Python 3: Issue 1265.

Comments (0)

Files changed (1)

 	return "\n".join(accu+all_deco)
 
 def sfilter(path):
-	if sys.version_info[0] >= 3 and Options.options.strip_comments:
-		f = open(path, "rb")
-		try:
-			tk = tokenize.tokenize(f.readline)
-			next(tk) # the first one is always tokenize.ENCODING for Python 3, ignore it
-			cnt = process_tokens(tk)
-		finally:
-			f.close()
-	elif Options.options.strip_comments and path.endswith('.py'):
-		f = open(path, "r")
-		try:
-			cnt = process_tokens(tokenize.generate_tokens(f.readline))
-		finally:
-			f.close()
+
+	if path.endswith('.py') :
+		if Options.options.strip_comments:
+			if sys.version_info[0] >= 3:
+				f = open(path, "rb")
+				try:
+					tk = tokenize.tokenize(f.readline)
+					next(tk) # the first one is always tokenize.ENCODING for Python 3, ignore it
+					cnt = process_tokens(tk)
+				finally:
+					f.close()
+			else:
+				f = open(path, "r")
+				try:
+					cnt = process_tokens(tokenize.generate_tokens(f.readline))
+				finally:
+					f.close()
+		else:
+			f = open(path, "r")
+			try:
+				cnt = f.read()
+			finally:
+				f.close()
+		# WARNING: since we now require python 2.4, we do not process the decorators anymore
+		# if you need such a thing, uncomment the code below:
+		#cnt = process_decorators(cnt)
+		#if cnt.find('set(') > -1:
+		#	cnt = 'import sys\nif sys.hexversion < 0x020400f0: from sets import Set as set\n' + cnt
+		cnt = '#! /usr/bin/env python\n# encoding: utf-8\n# WARNING! Do not edit! http://waf.googlecode.com/git/docs/wafbook/single.html#_obtaining_the_waf_file\n\n' + cnt
+
 	else:
 		f = open(path, "r")
 		try:
 		finally:
 			f.close()
 
-	if path.endswith('.py') :
-		# WARNING: since we now require python 2.4, we do not process the decorators anymore
-		# if you need such a thing, uncomment the code below:
-		#cnt = process_decorators(cnt)
-		#if cnt.find('set(') > -1:
-		#	cnt = 'import sys\nif sys.hexversion < 0x020400f0: from sets import Set as set\n' + cnt
-		cnt = '#! /usr/bin/env python\n# encoding: utf-8\n# WARNING! Do not edit! http://waf.googlecode.com/git/docs/wafbook/single.html#_obtaining_the_waf_file\n\n' + cnt
-
 	if sys.hexversion > 0x030000f0:
 		return (io.BytesIO(cnt.encode('utf-8')), len(cnt), cnt)
 	return (io.BytesIO(cnt), len(cnt), cnt)
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.