Henning Schröder avatar Henning Schröder committed 698ba38

source index search engine

Comments (0)

Files changed (8)

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import inspect
+import textwrap
+import re
+from types import (
+   ModuleType, ClassType, ObjectType, MethodType, FunctionType, UnboundMethodType,
+   BuiltinFunctionType, BuiltinMethodType)
+from pprint import pformat
+
+
+
+
+class StubCreator(object):
+    
+    def __init__(self, module_name):
+        mod = __import__(module_name, {}, {}, [])
+        for name in module_name.split(".")[1:]:
+            mod = getattr(mod, name)
+        self.mod = mod
+        self.indention_level = 0
+        self.indention = " " * 4
+        self.code = [
+          "# -*- coding: utf-8 -*-",
+          "# generated stub module from %r" % self.mod.__file__.rstrip("co")
+        ]
+        doc = self.mod.__doc__
+        if doc:
+            self.emit('"""\n%s\n"""' % doc.rstrip("\n"))
+        self.emit()
+        self.generate(self.mod)
+
+        
+    def emit(self, line=""):
+        indent = self.indention * self.indention_level
+        if line:
+            line = indent + line
+        self.code.append(line)
+
+        
+    def indent(self, count=1):
+        self.indention_level += count
+        
+        
+    def dedent(self, count=1):
+        self.indention_level = max(0, self.indention_level - count)
+       
+        
+    def is_direct_instance(self, obj, *types):
+        for t in types:
+            if isinstance(obj, t) and obj.__class__ is t:
+                return True
+        return False
+
+        
+    def is_save(self, root):
+        if self.is_direct_instance(root, basestring, int, float, bool):
+            return True
+        elif self.is_direct_instance(root, tuple, list):
+            for item in root:
+                if not self.is_save(item):
+                    return False
+            return True
+        elif self.is_direct_instance(root, dict):
+            for key, value in root.items():
+                if not self.is_save(key):
+                    return False
+                if not self.is_save(value):
+                    return False
+            return True
+        else:
+            return False
+
+            
+        
+    def generate(self, root):
+        for name in dir(root):
+            if name == "__all__" and root is self.mod:
+                pass
+            elif name.startswith("_"):
+                continue
+            
+            try:
+                value = getattr(root, name)
+            except AttributeError, e:
+                print "!", e
+                continue
+
+            if not callable(value) and self.is_save(value):
+                # constant
+                self.emit("%s = %s" % (name, pformat(value)))
+
+            
+            elif isinstance(value, (FunctionType, MethodType, UnboundMethodType, BuiltinFunctionType, BuiltinMethodType)):
+                # function
+                if value.__name__ != name:
+                    self.emit("%s = %s" % (name, value.__name__))
+                    self.emit()
+                else:
+                    self.generate_function(value)
+
+            elif isinstance(value, ModuleType):
+                # module
+                if name != value.__name__:
+                    self.emit("import %s as %s" % (value.__name__, name))
+                else:
+                    self.emit("import %s" % name)
+                    
+            elif isinstance(value, (ClassType,ObjectType)):
+                # class
+                if hasattr(value, "__name__"):
+                    if value.__name__ != name:
+                        self.emit("%s = %s" % (name, value.__name__))
+                        self.emit()
+                    else:                        
+                        self.generate_class(value)
+
+
+                
+    def generate_class(self, cls):
+        bases = (b.__name__ for b in getattr(cls, "__bases__", []))
+        self.emit("class %s(%s):" % (cls.__name__, ", ".join(bases)))
+        self.indent()
+        doc = cls.__doc__
+        if not doc:
+            doc = ""
+        self.emit('"""%s"""' % doc)
+        self.generate(cls)
+        self.dedent()
+        self.emit()
+
+
+    def extract_signature_from_doc(self, doc):
+        lines = [l.strip() for l in (doc or "").splitlines() if l.strip()]
+        args = []
+        if self.indention_level > 0:
+            args.append("self")
+        for l in lines:
+            found = re.match(".*\((?P<args>.*)\)($|[ ]*->)", l)
+            if found:
+                self.emit("# real signature not available, using information from __doc__")
+                for a in found.group("args").split(","):
+                    a = a.strip()
+                    if "=" in a:
+                        a, _sep, v = a.partition("=")
+                        a = a.strip()
+                    else:
+                        v = None
+                    a = a.replace(" ", "_")
+                    if v and a:
+                        a = "%s=%s" % (a, v)
+                    if a == "...": # Found in PyQt
+                        a = "*args" 
+                    if a:
+                        args.append(a)
+                break
+        return "(%s)" % ", ".join(args)
+
+        
+    def generate_function(self, func):
+        doc = func.__doc__
+        try:
+            (args, varargs, varkw, defaults) = inspect.getargspec(func)
+            signature = inspect.formatargspec(args, varargs, varkw, defaults)
+        except TypeError, e:
+            signature = self.extract_signature_from_doc(doc)
+        if not doc:
+            doc = "%s%s" % (func.__name__, signature)
+        self.emit("def %s%s:" % (func.__name__, signature))
+        self.indent()
+        indent = self.indention * self.indention_level
+        self.emit('"""\n%s%s\n%s"""' % (indent, doc.rstrip("\n"), indent))
+        self.dedent()
+        self.emit()
+
+        
+    def __str__(self):
+        return "\n".join(self.code)
+
+        
+    
+        
+        
+s = StubCreator("PyQt4.QtGui")
+print s
+from pprint import pformat
+from inspect import isfunction, isbuiltin, getargspec, isclass, ismethod
+
+
+class StubCreator(object):
+
+    def __init__(self):
+        self.out = []
+        self.indent = " " * 4
+
+
+    def emit(self, code, depth=0):
+        self.out.append("%s%s" % (self.indent*depth, code))
+
+        
+    def emit_doc(self, doc, depth):
+        if doc:
+            if "\n" in doc:
+                doc = "\n".join(self.indent + l for l in doc.splitlines())[len(self.indent):]
+                self.emit("'''%s'''" % doc, depth)
+            else:
+                self.emit(pformat(doc), depth)
+            self.emit("")
+
+        
+    def __str__(self):
+        return "\n".join(self.out)
+
+
+    def create_stub(self, root, depth=0):
+        doc = getattr(root, "__doc__")
+        if doc:
+            self.emit_doc(doc, depth)
+        name_list = dir(root)
+        for name in name_list:
+            if name == "__doc__":
+                continue
+            try:
+                value = getattr(root, name)
+            except AttributeError, e:
+                continue
+            if isinstance(value, int) or isinstance(value, float) or \
+               isinstance(value, basestring) or value is None:
+                self.emit("%s = %r" % (name, value), depth)
+            elif callable(value):
+                if isfunction(value) or isbuiltin(value):
+                    self.create_function_stub(name, value, depth)
+                elif isclass(value):
+                    self.create_class_stub(name, value, depth)
+            else:
+                pass #print "*", name, type(value), callable(value)
+
+            
+    def create_class_stub(self, name, cls, depth):
+        self.emit("")
+        if cls.__name__ != name:
+            self.emit("%s = %s # class alias" % (name, cls.__name__), depth)
+        else:
+            bases = cls.__bases__
+            if bases:
+                bases = "(%s)" % ", ".join(b.__name__ for b in bases)
+            self.emit("class %s%s:" % (name, bases))                
+            if not self.create_stub(cls, depth+1):
+                self.emit("pass", depth+1)
+
+
+    def func_args(self, func):
+        try:
+            args, varargs, keywords, defaults = getargspec(func)
+            defaults = dict(zip(args[-len(defaults):], defaults))
+            if varargs:
+                args.append("*"+varargs)
+            for i, a in enumerate(args):
+                try:
+                    d = defaults[a]
+                    args[i] = "%s=%r" % (a, d)
+                except KeyError, e:
+                    pass
+            return ", ".join(args)
+        except TypeError, e:
+            doc = getattr(func, "__doc__", None)
+            if doc:
+                l = func.__doc__.splitlines()[0].strip()
+                args, sep, result = l.partition("->")
+                return "self, " + args.split("(", 1)[1].rsplit(")", 1)[0]
+            return "*args"
+
+                
+    def create_function_stub(self, name, func, depth):
+        self.emit("")
+        if func.__name__ != name:
+            self.emit("%s = %s # function alias" % (name, func.__name__), depth)
+        else:
+            args = self.func_args(func)
+            self.emit("def %s(%s):" % (name, args), depth)
+            doc = getattr(func, "__doc__", "")
+            if doc:
+                self.emit_doc(doc, depth+1)
+            else:
+                self.emit("pass", depth+1)
+                self.emit("")
+        self.emit("")
+
+        
+    def create_stub_from_module(self, module_name):
+        mod = __import__(module_name, {}, {}, [])
+        for part in module_name.split(".")[1:]:
+            mod = getattr(mod, part)
+        doc = getattr(mod, "__doc__")
+        if doc:
+            self.emit_doc(doc)
+        self.create_stub(mod)
+        
+    
+c = StubCreator()
+c.create_stub_from_module("PyQt4.QtGui")
+print c

sourceindex/index.py

+import os
+from whoosh.index import open_dir, create_in
+import models
+from parser import parse
+
+
+class Indexer(object):
+	
+	def __init__(self, path):
+		if not os.path.exists(path):
+			os.makedirs(path)
+		try:
+			self._ix = open_dir(path)
+		except:
+			self._ix = create_in(path, models.name_schema)
+		self._writer = self._ix.writer()
+
+	
+	def close(self):
+		self._writer.commit()
+
+	
+	def add(self, **kwargs):
+		for key, value in kwargs.items():
+			if isinstance(value, str):
+				kwargs[key] = unicode(value)
+		self._writer.add_document(**kwargs)
+		#self._writer.commit()
+
+		
+if __name__ == "__main__":
+	idx = Indexer("index")
+	for entry in parse("string"):
+		kind, scope, name, lineno = entry
+		idx.add(kind=kind, name=name, line=lineno)
+	idx.close()

sourceindex/models.py

+from whoosh.index import create_in
+from whoosh.fields import Schema, TEXT, ID, KEYWORD, NUMERIC
+
+
+# for fulltext search in comments and doc-strings
+comments_schema = Schema(
+	filename=ID(stored=True), 
+	comments=TEXT)
+
+
+# for name search
+name_schema = Schema(
+	filename=ID(stored=True), 
+    name=TEXT(stored=True), 
+	kind=KEYWORD(stored=True), 
+	scope=KEYWORD(stored=True), 
+	line=NUMERIC(stored=True)
+	)

sourceindex/parser.py

+import inspect
+
+
+def parse_dynamic(module_name):
+
+	def parse_scope(obj, scope):
+		
+		def parse_args(obj, lineno, scope):
+			argspec = inspect.getargspec(obj)
+			for arg in argspec.args:
+				yield ("argument", ".".join(scope), arg, lineno)
+		
+		def findsource(obj):
+			try:
+				code, lineno = inspect.findsource(obj)
+			except IOError:
+				code = lineno = ""
+			except TypeError:
+				code = lineno = ""
+			return code, lineno
+	
+		doc = inspect.getdoc(obj) or inspect.getcomments(obj)
+		for (name, member) in inspect.getmembers(obj):
+			if inspect.isfunction(member):
+				code, lineno = findsource(member)
+				yield ("func", ".".join(scope), name, lineno)
+				for result in parse_args(member, lineno, scope + [name]):
+					yield result
+			elif inspect.isclass(member) and  member.__module__ == scope[0]:
+				code, lineno = findsource(member)
+				yield ("class", ".".join(scope), name, lineno)
+				for result in parse_scope(member, scope + [name]):
+					yield result
+			elif inspect.ismethod(member):
+				code, lineno = findsource(member)
+				yield ("method", ".".join(scope), name, lineno)
+				for result in parse_args(member, lineno, scope + [name]):
+					yield result
+	
+	mod = __import__(module_name, {}, {}, ["foobardoesnotexist"])
+	for result in parse_scope(mod, [mod.__name__]):
+		yield result
+	
+
+
+parse = parse_dynamic
+
+
+
+if __name__ == "__main__":
+	for item in parse("string"):
+		print item

sourceindex/search.py

+from whoosh.qparser import QueryParser
+from whoosh.index import open_dir
+
+
+class Search(object):
+
+	
+	def __init__(self, path):
+		self._ix = open_dir(path)
+
+
+	def search(self, qs):
+		with self._ix.searcher() as searcher:
+			query = QueryParser("name", self._ix.schema).parse(qs)
+			results = searcher.search(query)
+			for hit in results:
+				yield hit
+
+
+				
+if __name__ == "__main__":
+	s = Search("index")
+	for hit in s.search("line:130 OR template"):
+		print hit
 # -*- coding: utf-8 -*-
 import sys
 from PyQt4.QtCore import Qt
-from PyQt4.QtGui import QApplication, QPlainTextEdit, QMainWindow
+from PyQt4.QtGui import QApplication, QPlainTextEdit, QMainWindow, QFont
 
 from pycode.project import Project
 from pycode.qtintegration import PyCode
 
+sys.path.append("syntaxhighlighter")
+try:
+    from highlighter import SyntaxHighlighter, load_syntax
+    from highlighter.python27 import syntax
+except Importerror:
+    SyntaxHighlighter = None
+    
+scheme = {
+  "syntax_comment": dict(color="green", italic=True),
+  "syntax_string": "magenta",
+  "syntax_builtin": "red",
+  "syntax_keyword": ("darkred", True),
+  "syntax_number": "blue",
+}
 
-        
 
 
 
     app = QApplication(sys.argv)
     win = QMainWindow()
     edit = QPlainTextEdit(None)
+    
+    font = QFont()
+    font.setFamily("Courier")
+    font.setPointSize(10)
+
+    edit.setFont(font)
+    if SyntaxHighlighter:
+        parts_scanner, code_scanner, formats = load_syntax(syntax, scheme)
+        hl = SyntaxHighlighter(edit.document(),
+          parts_scanner, code_scanner, formats, default_font=font)
+
+    
     pc = PyCode(".", edit)
     edit.setPlainText(open("testedit.py").read())
     win.setCentralWidget(edit)
+import sys
+import inspect
+import linecache
+
+
+class Parameters(object):
+    
+    def __init__(self):
+        self.types = None
+        self.rtype = None
+        
+    def __repr__(self):
+        return repr(self.__dict__)
+
+
+class Method(object):
+    
+    def __init__(self):
+        self.callers = []
+        self.defined_at = None
+        self.parameters = {}
+
+    def __repr__(self):
+        return repr(self.__dict__)
+
+    
+class Class(object):
+    
+    def __init__(self):
+        self.methods = {}
+
+
+
+
+class Tracer(object):
+    
+
+    def __init__(self):
+        self.classes = {None: {None:{}}}
+        self.current_call = []
+
+        
+    def enable(self):
+        sys.settrace(self.trace_it)
+
+        
+    def disable(self):
+        sys.settrace(None)
+
+
+    def trace_it(self, frame, event, arg):
+        # http://docs.python.org/library/inspect.html
+        if event == "line":
+            lineno = frame.f_lineno
+            filename = frame.f_globals["__file__"]
+            if filename == "<stdin>":
+                filename = "__file__"
+            if (filename.endswith(".pyc") or filename.endswith(".pyo")):
+                filename = filename[:-1]
+            name = frame.f_globals["__name__"]
+            line = linecache.getline(filename, lineno)
+            #print "LINE %s:%d: %s" % (name, lineno, line.rstrip())
+        elif event in ("call", "c_call"):
+            arginfo  = inspect.getargvalues(frame)
+            if arginfo.args and arginfo.args[0] == "self":
+                # assume method
+                self.trace_method(frame, arginfo)
+            else:
+                self.current_call.append(None)
+        elif event in ("return", "c_return"):
+            self.trace_return(frame, arg)
+        elif event in ("exception", "c_exception"):
+            (exception, exc_value, exc_traceback) = arg
+            print "RAISE", arg
+        else:
+            print "EVENT is", event
+        return self.trace_it
+
+    
+    def trace_return(self, frame, result):
+        params = self.current_call.pop()
+        if params is not None:
+            params.rtype = type(result).__name__
+
+    
+    def trace_method(self, frame, arginfo):
+        line = frame.f_lineno
+        arg_names, arg_values = arginfo.args, arginfo.locals
+        obj = arginfo.locals["self"]
+        cls_name = obj.__class__.__name__
+        method_name = frame.f_code.co_name
+        arg_types = []
+        for arg in arg_names:
+            aval = arg_values[arg]
+            arg_types.append(type(aval).__name__)
+        cls = self.classes.get(cls_name)
+        if cls is None:
+            cls = self.classes[cls_name] = Class()
+        arg_names = tuple(arg_names[1:])
+        arg_types = tuple(arg_types[1:])
+        pkey = (arg_names, arg_types)
+        method = cls.methods.get(method_name)
+        if method is None:
+            method = cls.methods[method_name] = Method()
+        params = method.parameters.get(pkey)
+        if params is None:
+            params = method.parameters[pkey] = Parameters()
+            method.defined_at = line
+        method.callers.append(frame.f_back.f_lineno)
+        self.current_call.append(params)
+
+
+def test_func(param):
+#    print param
+    return param
+
+
+class TestObj(object):
+
+    def __init__(self):
+        pass
+
+    def test_method(self, mp=123):
+        return mp * 2
+
+
+def main():
+    foo = "bar"
+    #test_func(1)
+    o = TestObj()
+    o.test_method(4)
+    o.test_method(mp=5.6)
+
+t = Tracer()
+t.enable()
+main()
+t.disable()
+import pprint
+for cls_name, cls in t.classes.items():
+    if cls_name is None:
+        continue
+    print "class", cls_name, cls
+    for method_name, method in cls.methods.items():
+        for (arg_names, arg_types), param in method.parameters.items():
+            print "\tdef", method_name, zip(arg_names, arg_types), "->", param.rtype
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.