Jan-Jaap Driessen avatar Jan-Jaap Driessen committed 2722e71 Merge

Comments (0)

Files changed (6)

doc/configuration.rst

 information from this. If no version information can be found or if
 the python package is installed in `development mode`_, we still want
 to be able to create a unique version that changes whenever the
-content of the resources changes. To this end, a hash of the contents
-of the Library directory is automatically calculated. Whenever you
-make any changes to a resource in the library, the hash version will
-be automatically recalculated.
+content of the resources changes.
+
+To this end, the most recent modification time from the files and directories
+in the Library directory is taken. Whenever you make any changes to a resource
+in the library, the hash version will be automatically recalculated.
 
 The benefit of calculating a hash for the Library directory is that
 resource URLs change when a referenced resource changes; If resource A
 because A changed, but because the contents of the library to which A
 and B belong has changed.
 
+Fanstatic also provides an MD5-based algorithm for the Library version
+calculation. This algorithm is slower, but you may use if you don't trust
+your filesystem. Use it through the ``versioning_use_md5`` parameter.
+
+
 .. _`development mode`: http://peak.telecommunity.com/DevCenter/setuptools#develop
 
 recompute_hashes

fanstatic/checksum.py

 import os
 import hashlib
+from datetime import datetime
 
 VCS_NAMES = ['.svn', '.git', '.bzr', '.hg']
 IGNORED_EXTENSIONS = ['.swp', '.tmp', '.pyc', '.pyo']
 
 
-def list_directory(path):
+def list_directory(path, include_directories=True):
     # Skip over any VCS directories.
     for root, dirs, files in os.walk(path):
         for dir in VCS_NAMES:
                 dirs.remove(dir)
             except ValueError:
                 pass
+        # We are also interested in the directories.
+        if include_directories:
+            yield os.path.join(root)
         for file in files:
             _, ext = os.path.splitext(file)
             if ext in IGNORED_EXTENSIONS:
             yield os.path.join(root, file)
 
 
-def checksum(path):
-    # Ignored extensions.
+def mtime(path):
+    latest = 0
+    for path in list_directory(path):
+        mtime = os.stat(path).st_mtime
+        latest = max(mtime, latest)
+    return datetime.fromtimestamp(latest).isoformat()[:22]
+
+def md5(path):
     chcksm = hashlib.md5()
-    for path in list_directory(path):
-        # Use the full path name for the checksum to track file renames.
+    for path in list_directory(path, include_directories=False):
         chcksm.update(path)
         try:
             f = open(path, 'rb')

fanstatic/config.py

 from fanstatic import DEBUG, MINIFIED
 
 BOOL_CONFIG = set(['versioning', 'recompute_hashes', DEBUG, MINIFIED,
-                   'bottom', 'force_bottom', 'bundle'])
+                   'bottom', 'force_bottom', 'bundle', 'versioning_use_md5'])
 
 
 def convert_config(config):

fanstatic/core.py

 import sys
 import threading
 
-from fanstatic.checksum import checksum
+import fanstatic.checksum
 
 DEFAULT_SIGNATURE = 'fanstatic'
 
                 'Resource path %s is already defined.' % resource.relpath)
         self.known_resources[resource.relpath] = resource
 
-    def signature(self, recompute_hashes=False):
+    def signature(self, recompute_hashes=False, version_method=None):
         """Get a unique signature for this Library.
 
         If a version has been defined, we return the version.
 
         if recompute_hashes:
             # Always re-compute.
-            sig = checksum(self.path)
+            sig = version_method(self.path)
         elif self._signature is None:
             # Only compute if not computed before.
-            sig = self._signature = checksum(self.path)
+            sig = self._signature = version_method(self.path)
         else:
             # Use cached value.
             sig = self._signature
       the URLs can both be infinitely cached and the resources will always
       be up to date. See also the ``recompute_hashes`` parameter.
 
+    :param versioning_use_md5: If ``True``, Fanstatic will use and md5
+      algorithm instead of an algorithm based on the last modification time of
+      the Resource files to compute versions. Use md5 if you don't trust your
+      filesystem.
+
     :param recompute_hashes: If ``True`` and versioning is enabled, Fanstatic
       will recalculate hash URLs on the fly whenever you make changes, even
       without restarting the server. This is useful during development,
 
     def __init__(self,
                  versioning=False,
+                 versioning_use_md5=False,
                  recompute_hashes=True,
                  bottom=False,
                  force_bottom=False,
             raise DeprecationWarning(
                 'Rollup has been superseded by bundling')
         self._versioning = versioning
+        if versioning_use_md5:
+            self._version_method = fanstatic.checksum.md5
+        else:
+            self._version_method = fanstatic.checksum.mtime
+
         self._recompute_hashes = recompute_hashes
         self._bottom = bottom
         self._force_bottom = force_bottom
         path.append(library.name)
         if self._versioning:
             path.append(
-                library.signature(recompute_hashes=self._recompute_hashes))
+                library.signature(
+                    recompute_hashes=self._recompute_hashes,
+                    version_method=self._version_method))
         return '/'.join(path)
 
     def render(self):

fanstatic/test_checksum.py

+import time
 import shutil
 from pkg_resources import resource_filename
 
-from fanstatic.checksum import list_directory, checksum
+from fanstatic.checksum import list_directory, md5, mtime
 from fanstatic.checksum import VCS_NAMES, IGNORED_EXTENSIONS
 
 
         tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
         tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
         ]
+    found = list(list_directory(testdata_path, include_directories=False))
+    assert sorted(found) == sorted(expected)
+
+    expected.extend([
+        tmpdir.join('MyPackage').strpath,
+        tmpdir.join('MyPackage/src').strpath,
+        tmpdir.join('MyPackage/src/mypackage').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources').strpath,
+    ])
     found = list(list_directory(testdata_path))
     assert sorted(found) == sorted(expected)
 
 
-def test_checksum(tmpdir):
-    testdata_path = str(_copy_testdata(tmpdir))
-    # As we cannot rely on a particular sort order of the directories,
-    # and files therein we cannot test against a given md5sum. So
-    # we'll have to do with circumstantial evidence.
-
-    # Compute a first checksum for the test package:
-    chksum_start = checksum(testdata_path)
-    # Add a file (+ contents!) and see the checksum changed:
-    tmpdir.join('/MyPackage/A').write('Contents for A')
-    assert checksum(testdata_path) != chksum_start
-
-    # Remove the file again, the checksum is same as we started with:
-    tmpdir.join('/MyPackage/A').remove()
-    assert checksum(testdata_path) == chksum_start
-
-    # Obviously, changing the contents will change the checksum too:
-    tmpdir.join('/MyPackage/B').write('Contents for B')
-    chksum_start = checksum(testdata_path)
-    tmpdir.join('/MyPackage/B').write('Contents for B have changed')
-    assert checksum(testdata_path) != chksum_start
-    tmpdir.join('/MyPackage/B').remove()
-
-    # Moving, or renaming a file should change the checksum:
-    chksum_start = checksum(testdata_path)
-    tmpdir.join('/MyPackage/setup.py').rename(
-        tmpdir.join('/MyPackage/setup.py.renamed'))
-    expected = [
-        tmpdir.join('MyPackage/setup.py.renamed').strpath,
-        tmpdir.join('MyPackage/MANIFEST.in').strpath,
-        tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
-        tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
-        ]
-    found = list(list_directory(testdata_path))
-    assert sorted(found) == sorted(expected)
-    assert checksum(testdata_path) != chksum_start
-
-
-def test_checksum_no_vcs_name(tmpdir):
+def test_list_directory_no_vcs_name(tmpdir):
     testdata_path = str(_copy_testdata(tmpdir))
     tmpdir.join('/MyPackage/.novcs').ensure(dir=True)
     tmpdir.join('/MyPackage/.novcs/foo').write('Contents of foo')
     expected = [
+        tmpdir.join('MyPackage').strpath,
+        tmpdir.join('MyPackage/.novcs').strpath,
         tmpdir.join('MyPackage/.novcs/foo').strpath,
         tmpdir.join('MyPackage/setup.py').strpath,
         tmpdir.join('MyPackage/MANIFEST.in').strpath,
+        tmpdir.join('MyPackage/src').strpath,
+        tmpdir.join('MyPackage/src/mypackage').strpath,
         tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources').strpath,
         tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
         ]
     found = list(list_directory(testdata_path))
     assert sorted(found) == sorted(expected)
 
 
-def test_checksum_vcs_name(tmpdir):
+def test_list_directory_vcs_name(tmpdir):
     testdata_path = str(_copy_testdata(tmpdir))
     for name in VCS_NAMES:
         tmpdir.join('/MyPackage/%s' % name).ensure(dir=True)
         tmpdir.join('/MyPackage/%s/foo' % name).write('Contents of foo')
         expected = [
+            tmpdir.join('MyPackage').strpath,
             tmpdir.join('MyPackage/setup.py').strpath,
             tmpdir.join('MyPackage/MANIFEST.in').strpath,
+            tmpdir.join('MyPackage/src').strpath,
+            tmpdir.join('MyPackage/src/mypackage').strpath,
             tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+            tmpdir.join('MyPackage/src/mypackage/resources').strpath,
             tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
             ]
         found = list(list_directory(testdata_path))
         tmpdir.join('/MyPackage/%s' % name).remove(rec=True)
 
 
-def test_checksum_dot_file(tmpdir):
+def test_list_directory_dot_file(tmpdir):
     testdata_path = str(_copy_testdata(tmpdir))
     tmpdir.join('/MyPackage/.woekie').ensure()
     expected = [
+        tmpdir.join('MyPackage').strpath,
         tmpdir.join('MyPackage/.woekie').strpath,
         tmpdir.join('MyPackage/setup.py').strpath,
         tmpdir.join('MyPackage/MANIFEST.in').strpath,
+        tmpdir.join('MyPackage/src').strpath,
+        tmpdir.join('MyPackage/src/mypackage').strpath,
         tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources').strpath,
         tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
         ]
     found = list(list_directory(testdata_path))
     assert sorted(found) == sorted(expected)
 
 
-def test_checksum_ignored_extensions(tmpdir):
+def test_list_directory_ignored_extensions(tmpdir):
     testdata_path = str(_copy_testdata(tmpdir))
     for ext in IGNORED_EXTENSIONS:
         tmpdir.join('/MyPackage/bar%s' % ext).ensure()
         expected = [
+            tmpdir.join('MyPackage').strpath,
             tmpdir.join('MyPackage/setup.py').strpath,
             tmpdir.join('MyPackage/MANIFEST.in').strpath,
+            tmpdir.join('MyPackage/src').strpath,
+            tmpdir.join('MyPackage/src/mypackage').strpath,
             tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+            tmpdir.join('MyPackage/src/mypackage/resources').strpath,
             tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
             ]
         found = list(list_directory(testdata_path))
         assert sorted(found) == sorted(expected)
+
+
+def test_mtime(tmpdir):
+    testdata_path = str(_copy_testdata(tmpdir))
+
+    # Compute a first mtime for the test package:
+    mtime_start = mtime(testdata_path)
+    # Add a file (+ contents!) and see the mtime changed:
+    tmpdir.join('/MyPackage/A').write('Contents for A')
+    mtime_after_add = mtime(testdata_path)
+    assert mtime_after_add != mtime_start
+
+    # Remove the file again, the mtime changed:
+    time.sleep(0.02) 
+    tmpdir.join('/MyPackage/A').remove()
+    mtime_after_remove = mtime(testdata_path)
+    assert mtime_after_remove != mtime_after_add
+    assert mtime_after_remove != mtime_start
+
+    # Obviously, changing the contents will change the mtime too:
+    tmpdir.join('/MyPackage/B').write('Contents for B')
+    mtime_start = mtime(testdata_path)
+    # Wait a split second in order to let the disk catch up.
+    time.sleep(0.02)
+    tmpdir.join('/MyPackage/B').write('Contents for B have changed')
+    assert mtime(testdata_path) != mtime_start
+    tmpdir.join('/MyPackage/B').remove()
+
+    # Moving, or renaming a file should change the mtime:
+    mtime_start = mtime(testdata_path)
+    time.sleep(0.02)
+    tmpdir.join('/MyPackage/setup.py').rename(
+        tmpdir.join('/MyPackage/setup.py.renamed'))
+    expected = [
+        tmpdir.join('MyPackage').strpath,
+        tmpdir.join('MyPackage/MANIFEST.in').strpath,
+        tmpdir.join('MyPackage/setup.py.renamed').strpath,
+        tmpdir.join('MyPackage/src').strpath,
+        tmpdir.join('MyPackage/src/mypackage').strpath,
+        tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
+        ]
+    found = list(list_directory(testdata_path))
+    assert sorted(found) == sorted(expected)
+    assert mtime(testdata_path) != mtime_start
+
+
+def test_md5(tmpdir):
+    testdata_path = str(_copy_testdata(tmpdir))
+
+    # Compute a first md5 for the test package:
+    md5_start = md5(testdata_path)
+    # Add a file (+ contents!) and see the md5 changed:
+    tmpdir.join('/MyPackage/A').write('Contents for A')
+    md5_after_add = md5(testdata_path)
+    assert md5_after_add != md5_start
+
+    # Remove the file again, the md5 is back to the previous one:
+    # This is a difference from the mtime approach!
+    tmpdir.join('/MyPackage/A').remove()
+    md5_after_remove = md5(testdata_path)
+    assert md5_after_remove != md5_after_add
+    assert md5_after_remove == md5_start
+
+    # Obviously, changing the contents will change the md5 too:
+    tmpdir.join('/MyPackage/B').write('Contents for B')
+    md5_start = md5(testdata_path)
+    # Wait a split second in order to let the disk catch up.
+    tmpdir.join('/MyPackage/B').write('Contents for B have changed')
+    assert md5(testdata_path) != md5_start
+    tmpdir.join('/MyPackage/B').remove()
+
+    # Moving, or renaming a file should change the md5:
+    md5_start = md5(testdata_path)
+    tmpdir.join('/MyPackage/setup.py').rename(
+        tmpdir.join('/MyPackage/setup.py.renamed'))
+    expected = [
+        tmpdir.join('MyPackage').strpath,
+        tmpdir.join('MyPackage/MANIFEST.in').strpath,
+        tmpdir.join('MyPackage/setup.py.renamed').strpath,
+        tmpdir.join('MyPackage/src').strpath,
+        tmpdir.join('MyPackage/src/mypackage').strpath,
+        tmpdir.join('MyPackage/src/mypackage/__init__.py').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources').strpath,
+        tmpdir.join('MyPackage/src/mypackage/resources/style.css').strpath,
+        ]
+    found = list(list_directory(testdata_path))
+    assert sorted(found) == sorted(expected)
+    assert md5(testdata_path) != md5_start
+

fanstatic/test_core.py

 from __future__ import with_statement
+import re
 import pytest
+import time
 
 from fanstatic import (Library,
                        Resource,
     foo = Library('foo', tmpdir.strpath)
 
     needed = NeededResources(versioning=True)
+    url = needed.library_url(foo)
+    assert re.match('/fanstatic/foo/:version:[0-9T:.-]*$', url)
 
-    assert (needed.library_url(foo) ==
-            '/fanstatic/foo/:version:d41d8cd98f00b204e9800998ecf8427e')
+    # The md5 based version URL is available through the
+    # `versioning_use_md5` parameter:
+    needed = NeededResources(versioning=True, versioning_use_md5=True)
+    md5_url = needed.library_url(foo)
+    assert url != md5_url
 
+    # If the Library defines a version, the version is used.
     bar = Library('bar', '', version='1')
-    assert (needed.library_url(bar) == '/fanstatic/bar/:version:1')
+    assert needed.library_url(bar) == '/fanstatic/bar/:version:1'
 
 
 def test_library_url_hashing_norecompute(tmpdir):
 
     # now create a file
     resource = tmpdir.join('test.js')
+    time.sleep(0.02)
     resource.write('/* test */')
 
     # the hash is recalculated now, so it changes
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.