Commits

Anonymous committed be7f7d7

#179 final housekeeping; check-in on branch

  • Participants
  • Parent commits 6ae3d92
  • Branches issue_0179_dwhall_unify_plat_dirs

Comments (0)

Files changed (2)

 MKDIR := mkdir -p
 TAGS := ctags
 CSCOPE := cscope
+PYCSCOPE := $(abspath src/tools/pycscope.py)
 PMDIST := src/tools/pmDist.py
 
 pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(PATH)))))
-PYCSCOPE := $(call pathsearch,pycscope.py)
 
 VPATH := . src/vm src/lib docs/src
 
 TAGS :
 	$(TAGS) -R *
 	$(CSCOPE) -b -c -R
-	$(if $(PYCSCOPE), cd src/tools && $(PYCSCOPE) -R *.py)
+	cd src/tools && $(PYCSCOPE) *.py
+	cd src/lib && $(PYCSCOPE) *.py
 
 dist :
 ifndef PM_RELEASE

File src/tools/pycscope.py

+#!/usr/bin/env python
+"""
+PyCscope
+
+PyCscope creates a Cscope-like index file for a tree of Python source.
+"""
+
+
+#
+# 2007/12/25:
+#   Improvements contributed by K. Rader of Google:
+#   - Added the `-i` argument to specify a file-list file
+#   - Fixups to the header and footer to make a valid file that cscope can read
+#
+
+
+__author__ = "Dean Hall"
+__copyright__ = "Copyright 2006 Dean Hall.  See LICENSE for details."
+__date__ = "2007/12/25"
+__version__ = "0.3"
+__usage__ = """Usage: pycscope.py [-R] [-f reffile] [-i srclistfile] [files ...]
+
+-R              Recurse directories for files.
+-f reffile      Use reffile as cross-ref file name instead of cscope.out.
+-i srclistfile  Use a file that contains a list of source files to scan."""
+
+
+import getopt, sys, os, os.path, string, types
+import keyword, parser, symbol, token
+
+# Marks as defined by Cscope
+MARK_FILE = "\t@"
+MARK_FUNC_DEF = "\t$"
+MARK_FUNC_CALL = "\t`"
+MARK_FUNC_END = "\t}"
+MARK_INCLUDE = "\t~<"   # TODO: assume all includes are global for now
+MARK_ASGN = "\t="
+MARK_CLASS = "\tc"
+MARK_GLOBAL = "\tg"
+MARK_FUNC_PARM = "\tp"
+
+# Reverse the key,value pairs in the token dict
+tok_name_lookup = dict((v,k) for k,v in token.tok_name.iteritems())
+TOK_NEWLINE = tok_name_lookup["NEWLINE"]
+TOK_NAME = tok_name_lookup["NAME"]
+TOK_LPAR = tok_name_lookup["LPAR"]
+TOK_ENDMARKER = tok_name_lookup["ENDMARKER"]
+TOK_INDENT = tok_name_lookup["INDENT"]
+TOK_DEDENT = tok_name_lookup["DEDENT"]
+
+# Reverse the key,value pairs in the symbol dict
+sym_name_lookup = dict((v,k) for k,v in symbol.sym_name.iteritems())
+SYM_TRAILER = sym_name_lookup["trailer"]
+SYM_VARARGSLIST = sym_name_lookup["varargslist"]
+
+# Get the list of Python keywords and add a few common builtins
+kwlist = keyword.kwlist
+kwlist.extend(("True", "False", "None", "object"))
+
+# Globals for the recursive walkAst function
+latestnewline = 1
+latestsymbol = ""
+latesttoken = ""
+prevtoken = ""
+mark = ""
+infuncdef = False
+indentcount = 0
+
+
+def main():
+    """Parse command line args and act accordingly.
+    """
+    # Parse the command line arguments
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "Rf:i:")
+    except getopt.GetoptError:
+        print __usage__
+        sys.exit(2)
+    recurse = False
+    indexfn = "cscope.out"
+    for o, a in opts:
+        if o == "-R":
+            recurse = True
+        if o == "-f":
+            indexfn = a
+        if o == "-i":
+            args.extend(map(string.rstrip, open(a, 'r').readlines()))
+
+    # Create the buffer to store the output (list of strings)
+    indexbuff = []
+    fnamesbuff = []
+
+    # Search current dir by default
+    if len(args) == 0:
+        args = "."
+
+    # Parse the given list of files/dirs
+    basepath = os.getcwd()
+    for name in args:
+        if os.path.isdir(os.path.join(basepath, name)):
+            parseDir(basepath, name, indexbuff, recurse, fnamesbuff)
+        else:
+            try:
+                parseFile(basepath, name, indexbuff, fnamesbuff)
+            except SyntaxError:
+                pass
+
+    # Symbol data for the last file ends with a file mark
+    indexbuff.append("\n" + MARK_FILE)
+    writeIndex(basepath, indexfn, indexbuff, fnamesbuff)
+
+
+def parseDir(basepath, relpath, indexbuff, recurse, fnamesbuff):
+    """Parses all files in the directory and
+    recurses into subdirectories if requested.
+    """
+    dirpath = os.path.join(basepath, relpath)
+    for name in os.listdir(dirpath):
+        fullpath = os.path.join(dirpath, name)
+        if os.path.isdir(fullpath) and recurse:
+            parseDir(basepath, os.path.join(relpath, name), indexbuff, recurse,
+                     fnamesbuff)
+        else:
+            try:
+                parseFile(basepath, os.path.join(relpath, name), indexbuff,
+                          fnamesbuff)
+            except SyntaxError:
+                pass
+
+
+def parseFile(basepath, relpath, indexbuff, fnamesbuff):
+    """Parses a source file and puts the resulting index into the buffer.
+    """
+    # Don't parse if it's not python source
+    if relpath[-3:] != ".py":
+        return
+
+    # Open the file and get the contents
+    fullpath = os.path.join(basepath, relpath)
+    f = open(fullpath, 'r')
+    filecontents = f.read()
+    f.close()
+
+    # Add the file mark to the index
+    fnamesbuff.append(relpath)
+    indexbuff.append("\n%s%s" % (MARK_FILE, relpath))
+    global latestnewline
+    latestnewline = len(indexbuff)
+
+    # Add path info to any syntax errors in the source files
+    try:
+        parseSource(filecontents, indexbuff)
+    except SyntaxError, se:
+        se.filename = fullpath
+        raise se
+
+
+def parseSource(sourcecode, indexbuff):
+    """Parses python source code and puts the resulting index into the buffer.
+    """
+    # Parse the source to an Abstract Syntax Tree
+    ast = parser.suite(sourcecode)
+    astlist = parser.ast2list(ast, True)
+
+    # Set these globals before each file's AST is walked
+    global sourcelinehassymbol
+    sourcelinehassymbol = False
+    global currentlinenum
+    currentlinenum = 0
+
+    # Walk the AST to index the rest of the file
+    walkAst(astlist, indexbuff)
+
+
+def walkAst(astlist, indexbuff):
+    """Scan the AST for tokens, write out index lines.
+    """
+    global latestnewline
+    global latestsymbol
+    global latesttoken
+    global prevtoken
+    global mark
+    global sourcelinehassymbol
+    global infuncdef
+    global indentcount
+    global currentlinenum
+
+    # Remember the latest symbol
+    if astlist[0] > 256:
+        latestsymbol = astlist[0]
+
+    # Handle the tokens
+    else:
+        # Save the previous token and get the latest one
+        prevtoken = latesttoken
+        latesttoken = astlist[0]
+
+        # If this code is on a new line number
+        if astlist[2] != currentlinenum:
+            currentlinenum = astlist[2]
+
+            # If there was a symbol of interest,
+            # remember this location in the index
+            if sourcelinehassymbol:
+                latestnewline = len(indexbuff)
+                sourcelinehassymbol = False
+
+            # If there was no symbol of interest between this and the previous
+            # newline, remove all entries added since the previous newline
+            else:
+                del indexbuff[latestnewline:]
+
+            # Write the new line number
+            indexbuff.append("\n\n%d " % astlist[2])
+
+            # Clear an include mark when a newline token is reached
+            # This is what ends a comma-separated list of modules after import
+            if mark == MARK_INCLUDE:
+                mark = ""
+
+        if latesttoken == TOK_NAME:
+            # If a name is not a python keyword, it is a symbol of interest
+            if astlist[1] not in kwlist:
+
+                # Remember that there is a symbol of interest
+                sourcelinehassymbol = True
+
+                # Write the mark and the symbol
+                indexbuff.append("\n%s%s\n" % (mark, astlist[1]))
+
+                # Clear the mark unless it's an include mark
+                # This is what allows a comma-separated list of modules after import
+                if mark != MARK_INCLUDE:
+                    mark = ""
+
+            # If the name is a python keyword
+            else:
+                # Some keywords determine what mark should prefix the next name
+                kw = astlist[1]
+                if kw == "def":
+                    mark = MARK_FUNC_DEF
+
+                    # Remember that we're in a function definition
+                    infuncdef = True
+                    indentcount = 0
+                elif kw == "import":
+                    mark = MARK_INCLUDE
+                elif kw == "class":
+                    mark = MARK_CLASS
+
+                # Write out the keyword
+                indexbuff.append("%s " % kw)
+
+        # This set of tokens and symbols indicates a function call (not perfect)
+        elif (latesttoken == TOK_LPAR) and (prevtoken == TOK_NAME) and (
+            (latestsymbol == SYM_TRAILER) or (latestsymbol == SYM_VARARGSLIST)):
+
+            # Insert a function-call mark before the previous name
+            indexbuff[-1] = "\n%s%s( " % (MARK_FUNC_CALL, indexbuff[-1][1:])
+
+        # Count the number of indents; to be used by dedent
+        elif latesttoken == TOK_INDENT:
+            if infuncdef:
+                indentcount += 1
+
+        # When dedent reaches the level of the function def,
+        # write the function-end mark
+        elif latesttoken == TOK_DEDENT:
+            if infuncdef:
+                indentcount -= 1
+                if indentcount == 0:
+                    indexbuff.insert(-1, "\n\n%d \n%s\n" % (astlist[2], MARK_FUNC_END))
+                    latestnewline += 1
+                    infuncdef = False
+
+        # Replace the last line number placeholder with a newline
+        # when at the end of a file
+        elif latesttoken == TOK_ENDMARKER:
+            if len(indexbuff) > 0:
+                indexbuff[-1] = "\n"
+
+        # For uninteresting tokens, just write the accompanying string
+        else:
+            if len(astlist[1]) > 0:
+                nonsymboltext = astlist[1].replace("\n","\\n") + ' '
+            else:
+                nonsymboltext = ''
+            indexbuff.append(nonsymboltext)
+
+    # Recurse into all nodes
+    for i in range(1, len(astlist)):
+        if type(astlist[i]) == types.ListType:
+            walkAst(astlist[i], indexbuff)
+
+
+def writeIndex(basepath, indexfn, indexbuff, fnamesbuff):
+    """Write the index buffer to the output file.
+    """
+    fout = open(os.path.join(basepath, indexfn), 'w')
+
+    # Write the header and index
+    index = ''.join(indexbuff)
+    index_len = len(index)
+    hdr_len = len(basepath) + 25
+    fout.write("cscope 15 %s -c %010d" % (basepath, hdr_len + index_len))
+    fout.write(index)
+
+    # Write trailer info
+    fnames = '\n'.join(fnamesbuff) + '\n'
+    fout.write("\n1\n.\n0\n")
+    fout.write("%d\n" % len(fnamesbuff))
+    fout.write("%d\n" % len(fnames))
+    fout.write(fnames)
+    fout.close()
+
+
+if __name__ == "__main__":
+    main()