Commits

Anonymous committed 3e066fe

Implement content signature caching and --max-drift (Anthony Roach)

Comments (0)

Files changed (10)

 .BR make .
 
 .TP
+.RI --max-drift= SECONDS
+Set the maximum expected drift in the modification time of files to 
+.IR SECONDS .
+This value determines how old a file must be before its content signature
+is cached. The default value is 2 days, which means a file must have a
+modification time of at least two days ago in order to have its content
+signature cached. A negative value means to never cache the content
+signature and to ignore the cached value if there already is one. A value
+of 0 means to always cache the signature, no matter how old the file is.
+
+.TP
 -n, --just-print, --dry-run, --recon
 No execute.  Print the commands that would be executed to build
 any out-of-date target files, but do not execute the commands.

src/engine/SCons/Node/FS.py

         else:
             return 0
 
-    def store_sigs(self):
-        """Update a file's .sconsign entry with its current info."""
-        self.dir.sconsign().set(self.name, self.get_timestamp(),
-                                self.get_bsig(), self.get_csig())
+    def store_csig(self):
+        old = self.get_prevsiginfo()
+        self.dir.sconsign().set(self.name,
+                                self.get_timestamp(),
+                                old[1],
+                                self.get_csig())
+
+    def store_bsig(self):
+        old = self.get_prevsiginfo()
+        self.dir.sconsign().set(self.name,
+                                self.get_timestamp(),
+                                self.get_bsig(),
+                                old[2])
 
     def get_prevsiginfo(self):
         """Fetch the previous signature information from the

src/engine/SCons/Node/NodeTests.py

         node.set_csig('zzz')
         assert node.get_csig() == 'zzz'
 
-    def test_store_sigs(self):
-        """Test calling the method to store signatures
+    def test_store_bsig(self):
+        """Test calling the method to store a build signature
         """
         node = SCons.Node.Node()
-        node.store_sigs()
+        node.store_bsig()
+
+    def test_store_sigs(self):
+        """Test calling the method to store a content signature
+        """
+        node = SCons.Node.Node()
+        node.store_csig()
 
     def test_set_precious(self):
         """Test setting a Node's precious value

src/engine/SCons/Node/__init__.py

             def get_parents(node, parent): return node.get_parents()
             def clear_cache(node, parent): 
                 node.implicit = None
+                node.bsig = None
             w = Walker(self, get_parents, ignore_cycle, clear_cache)
             while w.next(): pass
 
+        # clear out the content signature, since the contents of this
+        # node were presumably just changed:
+        self.csig = None
+
     def depends_on(self, nodes):
         """Does this node depend on any of 'nodes'?"""
         for node in nodes:
         of its dependency files and build information)."""
         self.bsig = bsig
 
+    def store_bsig(self):
+        """Make the build signature permanent (that is, store it in the
+        .sconsign file or equivalent)."""
+        pass
+
     def get_csig(self):
         """Get the signature of the node's content."""
         return self.csig
         """Set the signature of the node's content."""
         self.csig = csig
 
-    def store_sigs(self):
-        """Make the signatures permanent (that is, store them in the
+    def store_csig(self):
+        """Make the content signature permanent (that is, store it in the
         .sconsign file or equivalent)."""
         pass
 
+    def get_prevsiginfo(self):
+        """Fetch the previous signature information from the
+        .sconsign entry."""
+        return None
+
+    def get_timestamp(self):
+        return 0
+
     def set_precious(self, precious = 1):
         """Set the Node's precious value."""
         self.precious = precious

src/engine/SCons/Script/__init__.py

 target_top = None
 exit_status = 0 # exit status, assume success by default
 profiling = 0
+max_drift = None
 
 # utility functions
 
 	long = ['list-where'],
 	help = "Don't build; list files and where defined.")
 
+    def opt_max_drift(opt, arg):
+        global max_drift
+        try:
+            max_drift = int(arg)
+        except ValueError: 
+            raise UserError, "The argument for --max-drift must be an integer."
+
+    Option(func = opt_max_drift,
+        long = ['max-drift'],
+        arg = 'SECONDS',
+        help = "Set the maximum system clock drift to be SECONDS.")
+
     def opt_n(opt, arg):
 	SCons.Action.execute_actions = None
 	CleanTask.execute = CleanTask.show
     nodes = filter(lambda x: x is not None, map(Entry, targets))
 
     if not calc:
-        calc = SCons.Sig.Calculator(SCons.Sig.MD5)
+        if max_drift is None:
+            calc = SCons.Sig.Calculator(SCons.Sig.MD5)
+        else:
+            calc = SCons.Sig.Calculator(SCons.Sig.MD5, max_drift)
 
     taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc)
 

src/engine/SCons/Sig/SigTests.py

     def get_bsig(self):
         return self.bsig
 
+    def store_bsig(self):
+        pass
+
     def set_csig(self, csig):
         self.csig = csig
 
     def get_csig(self):
         return self.csig
 
+    def store_csig(self):
+        pass
+
     def get_prevsiginfo(self):
         return (self.oldtime, self.oldbsig, self.oldcsig)
-
+    
     def builder_sig_adapter(self):
         class Adapter:
             def get_contents(self):
         node.oldtime = node.file.timestamp
         node.oldbsig = calc.bsig(node)
         node.oldcsig = calc.csig(node)
-        
+
+def clear(nodes):
+    for node in nodes:
+        node.csig = None
+        node.bsig = None
 
 class SigTestBase:
     
         self.test_initial()
         self.test_built()
         self.test_modify()
+        self.test_modify_same_time()
         self.test_delete()
         self.test_cache()
         
         self.files[6].modify('blah blah blah', 333)
         self.files[8].modify('blah blah blah', 333)
 
+        clear(nodes)
+
         self.failUnless(not current(calc, nodes[0]), "modified directly")
         self.failUnless(not current(calc, nodes[1]), "direct source modified")
         self.failUnless(current(calc, nodes[2]))
         self.failUnless(not current(calc, nodes[9]), "direct source modified")
         self.failUnless(not current(calc, nodes[10]), "indirect source modified")
 
+
+    def test_modify_same_time(self):
+
+        nodes = create_nodes(self.files)
+
+        calc = SCons.Sig.Calculator(self.module, 0)
+
+        write(calc, nodes)
+
+        #simulate a modification of some files without changing the timestamp:
+        self.files[0].modify('blah blah blah blah', 333)
+        self.files[3].modify('blah blah blah blah', 333)
+        self.files[6].modify('blah blah blah blah', 333)
+        self.files[8].modify('blah blah blah blah', 333)
+
+        clear(nodes)
+
+        for node in nodes:
+            self.failUnless(current(calc, node),
+                            "all of the nodes should be current")
+
     def test_delete(self):
         
         nodes = create_nodes(self.files)
                 return 1
             def get_bsig(self):
                 return self.bsig
+            def set_bsig(self, bsig):
+                self.bsig = bsig
+            def store_sigs(self):
+                pass
             def get_csig(self):
                 return self.csig
             def get_prevsiginfo(self):

src/engine/SCons/Sig/__init__.py

 import os.path
 import string
 import SCons.Node
+import time
 
 #XXX Get rid of the global array so this becomes re-entrant.
 sig_files = []
         dir - the directory for the file
         module - the signature module being used
         """
-        
+
         self.dir = dir
         self.module = module
         self.sconsign = os.path.join(dir.path, '.sconsign')
         self.entries = {}
         self.dirty = None
-                    
+
         try:
             file = open(self.sconsign, 'rt')
         except:
         filename - the filename whose signature will be returned
         returns - (timestamp, bsig, csig)
         """
-        
+
         try:
             arr = map(string.strip, string.split(self.entries[filename], " "))
         except KeyError:
     for the build engine.
     """
 
-    def __init__(self, module):
+    def __init__(self, module, max_drift=2*24*60*60):
         """
         Initialize the calculator.
 
         module - the signature module to use for signature calculations
+        max_drift - the maximum system clock drift used to determine when to
+          cache content signatures. A negative value means to never cache
+          content signatures. (defaults to 2 days)
         """
         self.module = module
+        self.max_drift = max_drift
 
     def bsig(self, node):
         """
         """
         if not node.use_signature:
             return None
-        #XXX If configured, use the content signatures from the
-        #XXX .sconsign file if the timestamps match.
 
         bsig = node.get_bsig()
         if not bsig is None:
             return bsig
 
-        sigs = []
-        for child in node.children():
-            sigs.append(self.get_signature(child))
+        sigs = map(self.get_signature, node.children())
         if node.builder:
             sigs.append(self.module.signature(node.builder_sig_adapter()))
+        bsig = self.module.collect(filter(lambda x: not x is None, sigs))
 
-        return self.module.collect(filter(lambda x: not x is None, sigs))
-        
+        node.set_bsig(bsig)
+
+        # don't store the bsig here, because it isn't accurate until
+        # the node is actually built.
+
+        return bsig
+
     def csig(self, node):
         """
         Generate a node's content signature, the digested signature
         """
         if not node.use_signature:
             return None
-        #XXX If configured, use the content signatures from the
-        #XXX .sconsign file if the timestamps match.
+
         csig = node.get_csig()
         if not csig is None:
             return csig
-        
-        return self.module.signature(node)
-        
+
+        if self.max_drift >= 0:
+            info = node.get_prevsiginfo()
+        else:
+            info = None
+
+        mtime = node.get_timestamp()
+
+        if (info and info[0] and info[2] and info[0] == mtime):
+            # use the signature stored in the .sconsign file
+            csig = info[2]
+            # Set the csig here so it doesn't get recalculated unnecessarily
+            # and so it's set when the .sconsign file gets written
+            node.set_csig(csig)
+        else:
+            csig = self.module.signature(node)
+            # Set the csig here so it doesn't get recalculated unnecessarily
+            # and so it's set when the .sconsign file gets written
+            node.set_csig(csig)
+
+            if self.max_drift >= 0 and (time.time() - mtime) > self.max_drift:
+                node.store_csig()
+
+        return csig
+
     def get_signature(self, node):
         """
         Get the appropriate signature for a node.
 
         if not node.builder and node.get_timestamp() == oldtime:
             return 1
-        
+
         return self.module.current(newsig, oldbsig)

src/engine/SCons/Taskmaster.py

         if self.targets[0].get_state() == SCons.Node.executing:
             self.set_tstates(SCons.Node.executed)
             for t in self.targets:
-                t.store_sigs()
+                t.store_bsig()
                 t.built()
 
         self.tm.executed(self.node)
                 desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ")
                 raise SCons.Errors.UserError, desc
 
-            for child in children:
-                if not child.builder:
-                    # set the signature for non-derived files
-                    # here so they don't get recalculated over
-                    # and over again:
-                    child.set_csig(self.calc.csig(child))
-
             # Add non-derived files that have not been built
             # to the candidates list:
             def derived(node):

src/engine/SCons/TaskmasterTests.py

     def set_csig(self, csig):
         self.csig = csig
 
-    def store_sigs(self):
+    def store_csig(self):
+        pass
+
+    def store_bsig(self):
         pass
   
     

test/option--max-drift.py

+#!/usr/bin/env python
+#
+# Copyright (c) 2001, 2002 Steven Knight
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import os.path
+import os
+import string
+import sys
+import TestSCons
+
+python = sys.executable
+
+test = TestSCons.TestSCons()
+
+test.write('build.py', r"""
+import sys
+contents = open(sys.argv[2], 'rb').read()
+file = open(sys.argv[1], 'wb')
+file.write(contents)
+file.close()
+""")
+
+test.write('SConstruct', """
+B = Builder(name = "B", action = r'%s build.py $TARGETS $SOURCES')
+env = Environment(BUILDERS = [B])
+env.B(target = 'f1.out', source = 'f1.in')
+env.B(target = 'f2.out', source = 'f2.in')
+""" % python)
+
+test.write('f1.in', "f1.in\n")
+test.write('f2.in', "f2.in\n")
+
+
+
+test.run(arguments = 'f1.out')
+
+test.run(arguments = 'f1.out f2.out', stdout =
+"""scons: "f1.out" is up to date.
+%s build.py f2.out f2.in
+""" % python)
+
+atime = os.path.getatime(test.workpath('f1.in'))
+mtime = os.path.getmtime(test.workpath('f1.in'))
+
+test.run(arguments = '--max-drift=0 f1.out f2.out', stdout =
+"""scons: "f1.out" is up to date.
+scons: "f2.out" is up to date.
+""")
+
+test.write('f1.in', "f1.in delta\n")
+os.utime(test.workpath('f1.in'), (atime,mtime))
+
+test.run(arguments = '--max-drift=0 f1.out f2.out', stdout =
+"""scons: "f1.out" is up to date.
+scons: "f2.out" is up to date.
+""")
+
+test.run(arguments = '--max-drift=-1 f1.out f2.out', stdout =
+"""%s build.py f1.out f1.in
+scons: "f2.out" is up to date.
+"""%python)
+
+test.pass_test()
+
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.