Commits

Olemis Lang committed 060ed55
  • Participants
  • Parent commits 58487af
  • Tags fsl_0.5.2

Comments (0)

Files changed (24)

-Copyright (c) 2005 Kristian Ovaska
+Copyright (c) 2005-2007 Kristian Ovaska
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 Metadata-Version: 1.0
 Name: fsl
-Version: 0.5.1
+Version: 0.5.2
 Summary: Specification language for selecting files from directory tree
 Home-page: http://www.cs.helsinki.fi/u/hkovaska/fsl/
 Author: Kristian Ovaska
 Author: Kristian Ovaska (kristian.ovaska [at] helsinki.fi) 
 WWW: http://www.cs.helsinki.fi/u/hkovaska/fsl/
-Version: 0.5.1
+Version: 0.5.2
 
 File Selection Language (FSL) is a descriptive language for
 file selection. It is used to selectively pick files from

File doc/bugs.txt

+************************************************************************
+OPEN BUGS
+************************************************************************
+
+Bud ID: 8
+Found: 2005-11
+Version(s): All
+Platform(s): All
+Severity: Low
+Status: Open
+Description: Lexer fails to raise a parse error on test file
+lexer_files\invalid_3. 
+
+
+************************************************************************
+CLOSED BUGS
+************************************************************************
+
 Bud ID: 1
 Found: 2005-11-21
 Version(s): 0.5
 Platform(s): All(?); at least Windows
+Severity: High
+Status: Fixed
 Description: When root directory is the root of file system, globs beginning 
 with * don't work
-Severity: High
 Details: The root directory for the rule got transformed into the current 
 directory (cwd) instead of the real root directory..
 
 Version(s): 0.5
 Platform(s): All
 Severity: Low
+Status: Fixed
 Description: make_struct.sh fails by raising exceptions
 Details: The fault is due to missing imports in tutils.py.
 
 Version(s): 0.5
 Platform(s): All(?)
 Severity: Low
+Status: Fixed
 Description: Test cases that check the modification time of a file don't work.
 Details: The test suite assumes a particular modification time for the file 
 "filea". However, the time may be off by several hours because of time zone 
 Version(s): 0.5
 Platform(s): All
 Severity: Low
+Status: Fixed
 Description: Interpreter.run raised other exceptions (such as WindowsError) than 
 the ones mentioned in documentation
+
+------------------------------------------------------------------------
+
+Bud ID: 5
+Found: 2006-01-07
+Version(s): 0.5 - 0.5.1
+Platform(s): All
+Severity: Low
+Status: Fixed
+Description: symboltable.Variable.__str__ called non-imported type2str function.
+
+------------------------------------------------------------------------
+
+Bud ID: 6
+Found: 2006-01-07
+Version(s): 0.5 - 0.5.1
+Platform(s): All
+Severity: Low
+Status: Fixed
+Description: fsltool.PrintHookHandler._format_filename raised a non-defined
+exception.
+
+------------------------------------------------------------------------
+
+Bud ID: 7
+Found: 2006-01-07
+Version(s): 0.5 - 0.5.1
+Platform(s): All
+Severity: Low
+Status: Fixed
+Description: Interpreter.Interpreter._has_exclusion_rule raised a non-defined
+exception.
+

File doc/changelog.html

 
 <h1>Change log for File Selection Language</h1>
 
+<h2>Version 0.5.2 (2007-03-05)</h2>
+
+<p>
+Minor bug fixes and additions to documentation.
+</p>
+
 <h2>Version 0.5.1 (2005-12-05)</h2>
 
 <p>
 <p>
 <hr>
 Up: <a href="index.html">FSL index</a><br>
-Updated 2005-12-04
+Updated 2007-03-05
 </body>
 </html>

File doc/index.html

 Author: <a href="http://www.cs.helsinki.fi/u/hkovaska/">Kristian Ovaska</a>
         (kristian.ovaska [at] helsinki.fi) <br>
 License: BSD<br>
-Version: 0.5.1 (2005-12-05)<br>
+Version: 0.5.2 (2007-03-05)<br>
 Requirements: Python 2.3 or greater<br>
 Platforms: Unix, Windows
 
 
 <p>
 <hr>
-Updated 2005-12-04
+Updated 2007-03-05
 </body>
 </html>

File doc/language.html

 <h1>File Selection Language Reference</h1>
 
 <p>
-Version 0.5.1 (2005-12-05)<br>
+Version 0.5.2 (2007-03-05)<br>
 Kristian Ovaska (kristian.ovaska [at] helsinki.fi)
 
 <h2>Contents</h2>
 <p>
 <hr>
 Up: <a href="index.html">FSL index</a><br>
-Updated 2005-11-04
+Updated 2007-03-05
 </body>
 </html>
+"""
+Abstract Syntax Tree (AST) node classes. The AST closely resembles
+the syntax of File Selection Language, so it's helpful to keep the
+language specification at hand.
+
+The AST classes are generally struct-like and don't validate
+their input values. 
+"""
+
 import os
 
 import globals
 }
 
 def type2str(expr_type):
+    """Return description string for expression type, which
+    is one of the constants globals.E_*. Raise ValueError
+    for invalid expr_type. None is accepted as input, however,
+    and means an unknown type."""
+    
     if expr_type is None:
         return '<UNKNOWN>'
     if expr_type not in _EXPR_TYPE_DICT:
 
 
 class ASTNode:
+    """
+    Abstract base class for AST nodes. An ASTNode instance
+    knows its parent node (which may be None for the root node)
+    and the root directory that is used for glob patterns.
+    
+    ASTNode implements the Visitor pattern with the visit method.
+    Visitor object are instances of the Visitor.Visitor class.
+    """
+    
     def __init__(self, parent, rootdir):
         if not rootdir:
             raise ValueError, 'Rootdir not set'
         _running_id_number += 1
 
     def visit(self, visitor, arg=None):
+        """Call the visit_XXX method corresponding to this
+        class on the visitor object."""
         class_name = self.__class__.__name__
         method = getattr(visitor, 'visit_' + class_name)
         return method(self, arg)
 ### Rules ##############################################################
 
 class Rule(ASTNode):
+    """Abstract base class for rule nodes."""
+    
     def __init__(self, parent, rootdir):
         ASTNode.__init__(self, parent, rootdir)
 
     def innerdir_common_prefix(self):
         """Return the common prefix of all innerdirs of the children
-        of the node."""
+        of the node. Innerdir is a property of glob patterns; see
+        documentation of GlobPattern for details."""
         raise NotImplementedError
 
 class BlockRule(Rule):
+    """
+    Abstract base class for block rules: rules that contain other
+    rules. The list of rules are stored in attribute 'rules'.
+    """
+    
     def __init__(self, parent, rootdir):
         Rule.__init__(self, parent, rootdir)
         self.rules = []
             parentstr = '%d' % self.parent.idnumber
         else:
             parentstr= '-'
-        return 'Block rule (length %d, id %s, parent %s, rootdir "%s")' % (len(self.rules), self.idnumber, parentstr, self.rootdir)
+        return 'Block rule (length %d, id %s, parent %s, rootdir "%s")' % (len(self.rules),
+            self.idnumber, parentstr, self.rootdir)
 
     def innerdir_common_prefix(self):
         children_prefixes = [c.innerdir_common_prefix() for c in self.rules]
 
 
 class IfBlockRule(BlockRule):
+    """
+    IF-block is a rule block guarded by an expression (if-expression).
+    The if-expression is stored in attribute 'if_expr' and is an
+    Expression instance.
+    """
+    
     def __init__(self, parent, rootdir):
         BlockRule.__init__(self, parent, rootdir)
         self.if_expr = None
             parentstr = '%d' % self.parent.idnumber
         else:
             parentstr= '-'
-        return 'IF-block rule (length %d, id %s, parent %s, rootdir "%s")' % (len(self.rules), self.idnumber, parentstr, self.rootdir)
+        return 'IF-block rule (length %d, id %s, parent %s, rootdir "%s")' % (len(self.rules),
+            self.idnumber, parentstr, self.rootdir)
 
 
 class SelectorRule(Rule):
+    """
+    Abstract base class for atomic (non-block) rules. These
+    rules select files for inclusion or exclusion. The boolean
+    attribute 'inclusive' marks whether a rule instance is inclusive
+    or exclusive.
+    """
+    
     def __init__(self, parent, rootdir, inclusive):
         Rule.__init__(self, parent, rootdir)
         self.inclusive = inclusive
 
 
 class GloblistRule(SelectorRule):
+    """
+    Glob-list is a list of glob rules. The attribute
+    'glob_patterns' stores the patterns; they are instances
+    of GlobPattern.
+    
+    The glob-list rule may have an optional if-expression.
+    It is stored as an Expression instance in the 'if_expr'
+    attribute and it is None if there is no if-expression
+    associated with the glob-list rule. 
+    """
+    
     def __init__(self, parent, rootdir, inclusive):
         SelectorRule.__init__(self, parent, rootdir, inclusive)
         self.glob_patterns = []
             exclstr = ''
         else:
             exclstr = 'EXCLUDE '
-        return 'Glob list %s(length %d, parent %s, rootdir "%s")' % (exclstr, len(self.glob_patterns), parentstr, self.rootdir)
+        return 'Glob list %s(length %d, parent %s, rootdir "%s")' % (exclstr, 
+            len(self.glob_patterns), parentstr, self.rootdir)
 
     def innerdir_common_prefix(self):
         children_prefixes = [c.innerdir for c in self.glob_patterns]
 
 
 class ForEachRule(SelectorRule):
+    """
+    For-each rule contains a list of glob patterns and an
+    expression that is evaluated for each file (in the interpreter).
+    The glob patterns are stored in attribute 'glob_patterns' and
+    the if-expression in 'if_expr'. There is also a variable expression,
+    in attribute 'variable_expr', that is an instance of VariableExpression.
+    """
+    
     def __init__(self, parent, rootdir, inclusive):
         SelectorRule.__init__(self, parent, rootdir, inclusive)
         self.variable_expr = None
             exclstr = ''
         else:
             exclstr = 'EXCLUDE '
-        return 'For each %s(variable "%s", %s globs, rootdir "%s")' % (exclstr, self.variable_expr.variable_name, len(self.glob_patterns), self.rootdir)
+        return 'For each %s(variable "%s", %s globs, rootdir "%s")' % (exclstr,
+            self.variable_expr.variable_name, len(self.glob_patterns), self.rootdir)
 
     def innerdir_common_prefix(self):
         children_prefixes = [c.innerdir for c in self.glob_patterns]
 
 
 class GlobPattern(ASTNode):
+    """
+    A single glob pattern, the most basic rule-level unit.
+    
+    Attributes:
+      - 'pattern' is the glob pattern as read from source file. It is
+         normalized by replacing backslashes with forward slashes.
+      - 'recursive' tells whether the pattern should be evaluated
+         in subdirectories recursively.
+      - 'is_absolute' tells whether the pattern is relative (to rootdir)
+        or absolute.
+      - 'fullglob' is an absolute version of the pattern, i.e. it begins
+        with a full directory specification. Like 'pattern', 'fullglob'
+        is normalized to forward slases.
+      - 'innerdir' is the innermost directory where the pattern should
+        be evaluated. In other word, the evaluation start here. Like
+        'fullglob', 'innerdir' is an absolute file name and normalized.
+    """
+    
     def __init__(self, parent, rootdir, pattern, recursive):
         ASTNode.__init__(self, parent, rootdir)
         self.pattern = pattern.replace('\\', '/')
 ### Expressions ########################################################
 
 class Expression(ASTNode):
+    """
+    Abstract base class for expressions. Expression type
+    (a constant globals.E_*) is stored in attribute 'expr_type'.
+    """
+    
     def __init__(self, parent, rootdir, expr_type):
         ASTNode.__init__(self, parent, rootdir)
         self.expr_type = expr_type
 
 
 class AndExpression(Expression):
+    """
+    Boolean AND expression type. The expressions are stored in
+    attributes 'expr1' and 'expr2' and are instances of Expression.
+    """
+    
     def __init__(self, parent, rootdir):
         Expression.__init__(self, parent, rootdir, globals.E_BOOLEAN)
         self.expr1 = None
 
 
 class OrExpression(Expression):
+    """
+    Boolean OR expression type. The expressions are stored in
+    attributes 'expr1' and 'expr2' and are instances of Expression.
+    """
+    
     def __init__(self, parent, rootdir):
         Expression.__init__(self, parent, rootdir, globals.E_BOOLEAN)
         self.expr1 = None
 
 
 class NotExpression(Expression):
+    """
+    Boolean NOT expression type. The expression to be negated is stored in
+    attribute 'expr' and is an instance of Expression.
+    """
+    
     def __init__(self, parent, rootdir,):
         Expression.__init__(self, parent, rootdir, globals.E_BOOLEAN)
         self.expr = None
 
 
 class FunctionExpression(Expression):
+    """
+    Function call expression. The name of the function is stored
+    as a string in 'funcname' and the paremeters in 'params'. The
+    parameters are instances of Expression. 
+    """
+    
     def __init__(self, parent, rootdir, funcname):
         Expression.__init__(self, parent, rootdir, None)
         self.funcname = funcname
 
 
 class CompareExpression(Expression):
+    """
+    Comparison operator expression. The expressions to be compared
+    are stored in attributes 'expr1' and 'expr2' and are instances of
+    Expression. The comparison operator is in 'operator_type' and
+    is a token type defined in lexer.T_*. 
+    """
+    
     def __init__(self, parent, rootdir, expr1, operator_type):
         Expression.__init__(self, parent, rootdir, globals.E_BOOLEAN)
         self.expr1 = expr1
 
 
 class LiteralExpression(Expression):
+    """
+    Number, string, filename or datetime literal. The string value
+    is stored in 'strvalue'. For datetime literals, the datetime value
+    is stored as an datetime.datetime instance in 'datetime_value'.
+    For filename literals, the value is stored in 'filename_value'. 
+    """
+    
     def __init__(self, parent, rootdir, strvalue, atom_type):
+        """
+        Init.
+        'atom_type' is an globals.E_* type that marks the type
+        of the literal.
+        """
+        
         Expression.__init__(self, parent, rootdir, atom_type)
         self.strvalue = strvalue
         self.datetime_value = None
 
 
 class VariableExpression(Expression):
+    """
+    Reference to a variable, as used in an expression. The variable
+    name in stored in 'variable_name' and a reference to an
+    symboltable.Variable instance is stored in 'variable_obj.
+    The value of the variable is accessed via 'variable_obj'.
+    """
+    
     def __init__(self, parent, rootdir, variable_name):
         Expression.__init__(self, parent, rootdir, None)
         self.variable_name = variable_name

File fsl/ASTChecker.py

     return utils.abspath(filename) == filename and '\\' not in filename
 
 class ASTChecker(Visitor.Visitor):
+    """
+    Visitor for checking an AST for various internal errors. The
+    errors checked by ASTChecker should not occur unless there is
+    a bug in the program.
+    """
+    
     def __init__(self):
         self.root_node_seen = False
 

File fsl/ASTLinearizer.py

 import Visitor
 
 class ASTLinearizer(Visitor.Visitor):
-    """Walk the AST in depth-first order and return linearized list
-    of SeletorRule instances."""
+    """
+    Walk the AST in depth-first order and return linearized list
+    of SeletorRule instances. Other node types are ignored.
+    This visitor should be called for a root-level rule node.
+    """
 
     def visit_BlockRule(self, ast, arg):
         li = []

File fsl/ASTPrinter.py

 import Visitor
 
 class ASTPrinter(Visitor.Visitor):
+    """
+    Visitor for printing an AST for debugging purposes.
+    """
+    
     def __init__(self, indent_amount=4):
         self.level = 0
         self.indent_amount = indent_amount

File fsl/ExpressionEval.py

 import utils
 
 class ExpressionEval(Visitor.Visitor):
+    """
+    Visitor that is used to evaluate the value of expressions.
+    Each visit_XXX method returns the value of the subexpression
+    that they handle. The supplemental argument 'arg' is not used.
+    
+    This class only handles the part of the AST tree that are
+    instances of Expression; instances of Rule have no meaning here.
+    
+    visit_XXX methods raise TypeError and ValueError exceptions
+    in case of errors.
+    """
 
     def visit_AndExpression(self, astnode, arg):
         true1 = astnode.expr1.visit(self)

File fsl/Interpreter.py

 callback functions that the interpreter calls on
 appropriate times. Hook handlers receive real-time
 information on interpretation and file-system scanning.
-Hook handlers are passed as parameter to the methods
+Hook handlers are passed as parameter to the functions
 mentioned above.
 
 About exceptions: the lexer, parser and interpreter
         self._hook_handlers = []
 
     def run(self):
+        """Interpret AST and return the resulting files as a list
+        of File.File instance."""
+        
         self._filedict = {}
         self._cur_rule_index = 0
 
             elif isinstance(rule, AST.ForEachRule):
                 match = self._check_eachrule_exclusion(rule, filename)
             else:
-                raise InternalError
+                raise RuntimeError
 
             if match:
                 return True  # Early exit
     ### Hooks ##########################################################
 
     def add_hook_handler(self, handler):
+        """Attach a hook handler to interpreter. The handler
+        must be a HookHandler.HookHandler instance."""
+        
         if not isinstance(handler, HookHandler.HookHandler):
             raise TypeError, 'handler must be instance of HookHandler'
         self._hook_handlers.append(handler)

File fsl/TypeChecker.py

+"""
+Visitor for AST type inference and type checking.
+"""
+
 import re
 
 import AST
 import Visitor
 import utils
 
-# Don't look on the next line.
-_DATETIME_RE = r'^\d{4}([-](\d{1,2}([-](\d{1,2}([ ](\d{1,2}([:](\d{1,2}([:](\d{1,2})?)?)?)?)?)?)?)?)?)?$'
-# Okay, I warned.
-
 def compare_compatible(type1, type2):
+    """Return whether values of types 'type1' and 'type2' can
+    be compared."""
     return ( ((type1 == type2) and (type1 != globals.E_BOOLEAN))
          or type1 == globals.E_INT and type2 == globals.E_FLOAT
          or type1 == globals.E_FLOAT and type2 == globals.E_INT
          or (type2 == globals.E_STRING_FILENAME_DATETIME and type1 in (globals.E_STRING, globals.E_FILENAME, globals.E_DATETIME)) )
 
 class TypeChecker(Visitor.Visitor):
-    """Infer and check expression types and decorate AST with type info.
+    """
+    Infer and check expression types and decorate AST with type info.
     Also construct symbol table and set the variable_obj field
     in VariableExpression instances. (The symbol table is not returned
-    itself.)"""
+    itself.)
+    """
 
     def __init__(self):
         self.symboltable = symboltable.SymbolTable()
             msg = 'Type of IF-expression must be boolean (is: %s)' % AST.type2str(astnode.if_expr.expr_type)
             raise globals.FSLSyntaxError(astnode.token, msg)
 
-#    def visit_GlobPattern(self, astnode, arg):
-#        raise NotImplementedError
-
-    # TODO: type checks for AND, OR, NOT
     def visit_AndExpression(self, astnode, arg):
         astnode.expr1.visit(self)
         astnode.expr2.visit(self)

File fsl/Visitor.py

 class Visitor:
+    """
+    Abstract base class for visitor objects. The visitor
+    mechanism is implemented by a combination of the AST.ASTNode.visit
+    method and this class.
+    
+    This base class defines skeleton methods that must be
+    overridden in concrete visitor classes. Each visit_XXX method
+    receives an astnode (AST.ASTNode instance) object as argument
+    and an additional arg argument that may be used to pass further
+    relevant information.
+    """
 
     def visit_BlockRule(self, astnode, arg):
         raise NotImplementedError

File fsl/fslbackup.py

+"""
+Command-line tool for making backups with FSL - not
+tested enough to be of production quality!
+"""
+
 import datetime
 import os
 import shutil
         self.size_stack = []  # (dirname, files, bytes)
         self.total_copiedbytes = 0
         self.total_copiedfiles = 0
-
-    """
-    def set_hooks(self, interpreter):
-        interpreter.set_include_file_hook(self.hook_include_file)
-        interpreter.set_start_scan_hook(self.hook_start_scan)
-        interpreter.set_end_scan_hook(self.hook_end_scan)
-    """
+        
+        self.errors = []
 
     def end(self):
         print
         print 'Total files copied: %s' % self.total_copiedfiles
         print 'Total bytes copied: %s' % utils.format_size(self.total_copiedbytes)
+        
+        if self.errors:
+            print
+            print '*** Errors ***'
+            for msg in self.errors:
+                print msg
+                
+        return len(self.errors) > 0
 
     def _calc_dest_filename(self, src_fileobj):
         absname =  src_fileobj.abs_filename.replace(':', '_')
     def include_file(self, fileobj, rule):
         dest_filename = self._calc_dest_filename(fileobj)
         if not self.dryrun:
-            self._copyfile(fileobj, dest_filename)
+            try:
+                self._copyfile(fileobj, dest_filename)
+            except (IOError, WindowsError), err:
+                msg = '(%s) %s' % (fileobj.abs_filename, err)
+                self.errors.append(msg)
+                print '* Error *', msg
 
     def start_scan(self, dirname, glob_pattern):
         pass
-#        print '%s' % dirname
 
     def end_scan(self, dirname, files_count, bytes_count):
         if bytes_count > self.stats_limit:
     hook_handler = BackupHookHandler(destdir, options.dryrun, 10000)
 
     interpreter = Interpreter.Interpreter(ast)
-#    hook_handler.set_hooks(interpreter)
     interpreter.add_hook_handler(hook_handler)
     interpreter.run()
-    hook_handler.end()
+    errors_occured = hook_handler.end()
+    
+    if errors_occured:
+        sys.exit(1)
+    else:
+        sys.exit(0)
 
 if __name__=='__main__':
     import sys

File fsl/fslparser.py

+"""
+Parser reads FSL source files and produces an AST from
+them.
+"""
+
 import cStringIO
 
 import functions
 _DEBUG = False
 
 class _Parser:
+    # This class is not usually used directly.
+    
     def __init__(self, ast, filename, rootdir):
         self.ast = ast   # BlockRule instance
         self.filename = filename
 
     ### Parser methods #################################################
 
-    # Couple helpers.
+    # Some helpers.
 
     def _parse_rule_sequence(self, parent, rootdir):
         """Parse rule sequence, return list of Rule instance."""
             raise globals.FSLParseError(self._curtoken, "Expected atom (STRING, INT, FLOAT or simple expression)")
 
 def parse(filenames, rootdir):
+    """
+    Parse each source file in 'filenames' and return
+    an AST.BlockRule instance. All the source files are combined
+    into the block-rule that is returned. 'rootdir' is the root
+    directory on the file system that is used for relative
+    globs.
+    """
+    
     rootdir = rootdir.replace('\\', '/')
     ast = AST.BlockRule(None, rootdir)
     if type(filenames) in (str, unicode):
     return ast
 
 def parse_string(string, rootdir):
+    """
+    Parse a string containing FSL source and return
+    an AST.BlockRule instance. 'rootdir' is the root
+    directory on the file system that is used for relative
+    globs.
+    """
+    
     rootdir = rootdir.replace('\\', '/')
     ast = AST.BlockRule(None, rootdir)
 

File fsl/fsltool.py

+"""
+Command-line tool. For command line usage, run the tool from command line
+with the --help parameter.
+"""
+
 from optparse import OptionParser
 import os
 import sys
 import utils
 
 class PrintHookHandler(HookHandler.HookHandler):
+    """
+    Hook handler for printing various information during file
+    system scanning.
+    """
+    
     def __init__(self, format_absolute, format_quote, print_details, print_stats,
             out_filename=None, stats_limit=1, printing_relative_dir=None):
+        """
+        Init.
+        
+        :param format_absolute: Print absolute filenames.
+        :param format_quote: Put quotes " around filenames.
+        :param print_details: Print detailed information.
+        :param print_stats: Print statistics about the files that are included.
+        :param out_filename: If set, all printing goes to the file specified.
+          If the file exists, it is erased. If None, printing goes to stdout.
+        :param stats_limit: Threshold in bytes print directory stats at scan time.
+        :param printing_relative_dir: When printing relative filenames, print
+          them relative to this directory. If None, use current working directory.
+        """
 
         self.format_absolute = format_absolute
         self.format_quote = format_quote
             filename = fileobj.calc_relative_filename(self.printing_relative_dir)
             if filename is None:
                 msg = "Can't print file '%s' as relative filename" % fileobj.abs_filename
-                raise Interpreter.FSLRuntimeError(msg)
+                raise globals.FSLRuntimeError(msg)
         if self.format_quote:
             filename = '"%s"'% filename
         return filename

File fsl/functions.py

+"""
+Implementation of FSL runtime functions such as 'age'
+and 'date'.
+"""
+
 import datetime
 import os
 import time
 import globals
 
 def f_age(filename):
+    """Return age of file as days."""
     modtime = os.path.getmtime(filename)
     age_seconds = time.time() - modtime
     return age_seconds / 86400.0
     return ''
 
 def f_date(filename):
+    """Return modification date of file as
+    datetime.datetime instance."""
     modtime = os.path.getmtime(filename)
     dt = datetime.datetime.fromtimestamp(modtime)
     return dt #.isoformat()
 
 def f_exists(filename):
+    """Return true if the file exists, false otherwise."""
     return os.path.exists(filename)
 
 def f_extract(timestamp, part):
+    """Extract a value (such as year) from timestamp, return
+    a numeric value. 'timestamp' must be a datetime.datetime
+    instance. 'part' must be one of 'year', 'month', 'day',
+    'hour', 'hours', 'minute', 'minutes', 'second', 'seconds',
+    'weekday', 'week'.
+    """
+    
     if not isinstance(timestamp, datetime.datetime):
         raise TypeError, 'timestamp must be datetime.datetime instance'
     part = part.lower()
         raise ValueError, 'Invalid part specifier: "%s"' % part
 
 def f_now():
+    """Return current system time as datetime.datetime instance."""
     return datetime.datetime.now() #.isoformat()
 
 def f_size(filename):
+    """Return size of files in bytes."""
     return os.path.getsize(filename)
 
 # 'name': ((params), output)
 }
 
 def get_function_return_type(funcname):
+    """Return the return type of given function, as a value
+    globals.E_*."""
     if funcname.lower() not in _FUNCTION_TYPES:
         raise ValueError, 'Unknown function: %s' % funcname
     return _FUNCTION_TYPES[funcname.lower()][2]
 
 def get_function_param_types(funcname):
+    """Return the parameter types of given function, as a
+    list of values globals.E_*."""
     if funcname.lower() not in _FUNCTION_TYPES:
         raise ValueError, 'Unknown function: %s' % funcname
     return _FUNCTION_TYPES[funcname.lower()][1]
 
 def execute_function(funcname, params):
+    """Execute FSL function, return the result of the function
+    call. 'params' is a list of parameter values; they are ordinary
+    Python values such as strings (instead of AST nodes or such).
+    Raise ValueError if 'funcname' is an invalid function or if
+    there is None in the parems list.
+    """
+    
     if funcname.lower() not in _FUNCTION_TYPES:
         raise ValueError, 'Unknown function: %s' % funcname
 

File fsl/globals.py

+"""
+Global constants and exceptions.
+"""
+
 # Expression types
 E_BOOLEAN = 1
 E_INT = 2

File fsl/lexer.py

+"""
+Lexer reads FSL source files and divides the source into
+atomic tokens.
+"""
+
 import re
 
 import fslparser
     T_OR: 'OR',
 }
 
-def type2str(s):
-    return _TYPE_STRINGS.get(s, '<UNKNOWN TOKEN TYPE>')
+def type2str(t):
+    """Return string describing the token type 't', which
+    must be one of T_* constants."""
+    return _TYPE_STRINGS.get(t, '<UNKNOWN TOKEN TYPE>')
 
 # Possible tests:
 # - no consecutive T_LINE_END tokens
 # - T_EOF token at end
 
 class Token:
+    """
+    Atomic syntax element in source file.
+    Each token has a type (stored in attribute 'tokentype', one
+    of T_* constants), string value (in 'strvalue'), source filename
+    where the token comes from (in 'filename') and line number (in 'lineno').
+    Numeric tokens have the numeric value in attribute 'numbervalue';
+    for others, the attribute is None.
+    """
+    
     def __init__(self, tokentype, strvalue, filename, lineno):
         self.tokentype = tokentype
         self.strvalue = strvalue
 _WHITESPACE = ' \t'
 
 def iter_tokens(filename):
+    """Generate (yield) tokens for source file 'filename'. The
+    tokens are instances of Token. Raise IOError if
+    filename can't be accessed."""
+    
     infile = file(filename, 'rt')
     for t in iter_tokens_fileobj(infile, filename):
         yield t
     infile.close()
 
 def iter_tokens_fileobj(infile, filename):
-    """Generate tokens for file object `infile`.
-    `Filename` is the source filename and is used for
+    """Generate (yield) tokens for file object 'infile'.
+    The tokens are instances of Token.
+    'Filename' is the source filename and is used only for
     error messages."""
 
-#    infile = file(filename, 'rt')
-
     line = ''
     eof = False
     lineno = 0
 
     assert indent_count == dedent_count
 
-#    infile.close()
     yield Token(T_EOF, '<EOF>', filename, lineno)
-
-
-def main(args):
-    if not args:
-        print 'lexer.py <filename>'
-        return
-
-    for filename in args:
-        for t in iter_tokens(filename):
-            print t
-
-if __name__=='__main__':
-    import sys
-    main(sys.argv[1:])

File fsl/symboltable.py

+"""
+Classes for runtime variables and parse-time symbol table.
+"""
+
+import AST
+
 class Variable:
+    """
+    Runtime variable with name, type and value.
+    The name is stored in 'variable_name', type in
+    'variable_type' (one of globals.E_*) and
+    value is 'value'.
+    """
+    
     def __init__(self, variable_name, variable_type):
         self.variable_name = variable_name
         self.variable_type = variable_type
         self.value = value
 
     def __str__(self):
-        return 'Variable (%s, type %s)' % (self.variable_name, type2str(self.variable_type))
+        return 'Variable (%s, type %s)' % (self.variable_name, AST.type2str(self.variable_type))
 
 
 class SymbolTable:
+    """
+    Stack-like 'symbol table' that is used during parsing. The
+    table is not used after the parsing phase.
+    """
+    
     def __init__(self):
-        self._stack = []
+        self._stack = [] # [Variable]
 
     def push(self, variable):
+        """Add a Variable instance to the top of the stack."""
+        # This gives O(n) instead of O(1)...
         self._stack.insert(0, variable)
 
     def pop(self):
+        """Return the top-most Variable instance."""
         self._stack.pop(0)
 
     def get(self, symbolname):
+        """Return the current (active) Variable instance for the
+        name 'symbolname'. If no such variable exists, return None."""
+        
         for variable in self._stack:
             if variable.variable_name == symbolname:
                 return variable

File fsl/utils.py

+"""
+Utility functions.
+"""
+
 import datetime
 import os
 import re
     return os.path.abspath(normpath).replace('\\', '/')
 
 def calc_fullglob_innerdir(rootdir, globpattern):
-    """Calculate absolute (full) glob pattern from relative pattern."""
+    """Calculate absolute (full) glob pattern and innerdir from
+    relative pattern. Return tuple fullglob, innerdir. See
+    AST.GlobPattern for the meaning of them."""
 
     rootdir = abspath(rootdir)
 
     return fullglob, innerdir
 
 def calc_innerdir(fullglob):
+    """Calculate innerdir from full glob pattern."""
+    
     fullglob = fullglob.replace('\\', '/')
     inner_end_index = 0
     for i, char in enumerate(fullglob):
     return innerdir
 
 def filename_is_absolute(pattern):
+    """Tell whether the pattern is an absolute or
+    relative pattern."""
+    
     norm_pattern = pattern.replace('\\', '/')
     if len(norm_pattern) >= 3 and norm_pattern[0].isalpha() and norm_pattern[1] == ':' and norm_pattern[2] == '/':
         # Windows file name
         return True
     return norm_pattern.startswith('/')
 
-
 def format_size(size):
+    """Return formatted string for file size. The
+    string has one of prefixes B, KB, MB, GB depending
+    on the size."""
+    
     if size < 1000:
         return '%d B' % size
     elif size < 1000000:
 def parse_datetime(string):
     """Parse datetime string, return `datetime.datetime` instance.
     Raise ValueError if the string can't be parsed."""
+    
     for pattern in _DATETIME_PATTERNS:
         match = re.match(pattern, string)
         if match:
         sys.exit(1)
 
     setup(name='fsl',
-          version='0.5.1',
+          version='0.5.2',
           description=SHORT_DESC,
           author='Kristian Ovaska',
           author_email='kristian.ovaska [at] helsinki.fi',