Henning Schröder avatar Henning Schröder committed 6c90aa3

indexing packages now works

Comments (0)

Files changed (2)

sourceindex/index.py

 import parser
 
 
+# TODO: add zipped packages
 
 class Indexer(object):
 
 			os.makedirs(path)
 		try:
 			self._ix = open_dir(path)
+			multisegment = False
 		except:
 			self._ix = create_in(path, models.name_schema)
-		self._writer = self._ix.writer()
+			multisegment = True
+		self._writer = self._ix.writer(procs=4, limitmb=128, multisegment=multisegment)
+		
+		#stem_ana = self._writer.schema["name"].format.analyzer
+		#stem_ana.cachesize = -1 # Set the cachesize to -1 to indicate unbounded caching
+		#stem_ana.clear()  # Reset the analyzer to pick up the changed attribute
 
-	
+
 	def close(self):
 		self._writer.commit()
 
 
 		
 	def remove_filename(self, filename):
+		# TODO: remove documents where file does not exist anymore
 		self._writer.delete_by_term("filename", filename)
-		
-		
+	
+			
 	def add_filename(self, filename, module_name):
 		try:
 			items = list(parser.parse(filename))
 		self.remove_filename(filename)
 		for entry in items:
 			kind, scope, name, lineno = entry
-			self._add(kind=kind, name=name, line=lineno, filename=filename)
+			self._add(kind=kind, name=name, line=lineno, filename=filename, scope=module_name + "." + scope)
 		self._add(kind="module", name=module_name, line=0, filename=filename)
 
 
+	def add_path(self, path):
+		for kind, filename, module_name in parser.modules(path):
+			yield filename
+			if kind == "mod":
+				self.add_filename(filename, module_name)
+			else: # kind == "pkg"
+				base = os.path.basename(filename)
+				for pkg_filename in parser.tree(filename):
+					pkg_mod = pkg_filename[len(filename)+1:].rstrip(".py")
+					pkg = os.path.join(base, pkg_mod).replace("/", ".")
+					self.add_filename(pkg_filename, pkg)
+
+				
+	def add_all(self):
+		for path in sys.path:
+			for entry in self.add_path(path):
+				yield entry
+		
+				
+	def add_stdlib(self):
+		for entry in self.add_path(parser.stdlib_path()):
+			yield entry
 
 		
 
 if __name__ == "__main__":
 	idx = Indexer("index")
-	for kind, filename, module_name in parser.modules(parser.stdlib_path()):
-		print filename
-		idx.add_filename(filename, module_name)
+	for found in idx.add_all():
+		print found
 	idx.close()

sourceindex/parser.py

 from rope.base import pycore, evaluate, pyobjects, project, resources
 from rope.base.pyobjectsdef import PyModule, PyPackage, PyClass
 		
-
+# TODO: add zipped packages
 
 def stdlib_path():
 	path = os.path.dirname(inspect.getsourcefile(inspect))
 	return path
 	
 
+def tree(top):
+	for dirpath, dirnames, filenames in os.walk(top):
+		if not os.path.exists(os.path.join(dirpath, "__init__.py")):
+			continue
+		for name in filenames:
+			if name.endswith(".py"):
+				yield os.path.join(dirpath, name)
+
+
 def modules(path):
-	for name in os.listdir(path):
-		filename = os.path.join(path, name)
-		if os.path.isdir(filename) and "-" not in name:
-			if os.path.exists(os.path.join(filename, "__init__.py")):
-				# skip check that __init__.py is not a directory
-				yield "pkg", filename, name
-		elif name.endswith(".py"):
-			yield "mod", filename, name[:-3]
-	
-			
+	if os.path.isdir(path):
+		for name in os.listdir(path):
+			filename = os.path.join(path, name)
+			if os.path.isdir(filename) and "-" not in name:
+				if os.path.exists(os.path.join(filename, "__init__.py")):
+					# skip check that __init__.py is not a directory
+					yield "pkg", filename, name
+			elif name.endswith(".py"):
+				yield "mod", filename, name[:-3]
+	else:
+		pass # .endswith("zip") ...
+
+
 def libraries(path):
 	dynload_path = os.path.join(path, "lib-dynload")
 	for name in os.listdir(path):
 	for item in parse_scope(pyobject.get_scope()):
 		yield item
 
+	
+def find_errors(pymodule):
+	from rope.base import ast
+	from rope.contrib.finderrors import  _BadAccessFinder
+    finder = _BadAccessFinder(pymodule)
+    ast.walk(pymodule.get_ast(), finder)
+    return finder.errors
+
+
 
 def parse_scope(scope):
 	kind = scope.get_kind()
 	print mods
 	fn = mods[-1][1]
 	print fn
-	for item in parse_rope(fn):
+	for item in parse(fn):
 		print item
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.