Commits

Jan-Jaap Driessen committed d38df83

Back-merge the rollup, dropping the eager-superseder as we go.

Comments (0)

Files changed (9)

doc/configuration.rst

 available. If no minified version is available, the default resource
 will be served.
 
+rollup
+------
+
+A performance optimization to reduce the amount of requests sent by a
+client is to roll up several resources into a bundle, so that all
+those resources are retrieved in a single request. This way a whole
+collection of resources can be served in one go.
+
+You can create special :py:class:`Resource` instances that declare
+they supersede a collection of other resources. If ``rollup`` is
+enabled, Fanstatic will serve a combined resource if it finds out that
+all individual resources that it supersedes are needed. If you also
+declare that a resource is an ``eager_superseder``, the rolled up
+resource will actually always be served, even if only some of the
+superseded resources are needed.
+
 base_url
 --------
 

doc/optimization.rst

   the mode system. You can then configure Fanstatic to preferentially
   serve resources in a certain mode, such as ``minified``.
 
+* rolling up of resources.  Resource libraries can specify rollup
+  resources that combine multiple resources into one. This reduces the
+  amount of server requests to be made by the web browser, and can
+  help with caching. This can be controlled with the ``rollup`` configuration
+  parameter.
+
 * bundling of resources.  Resource bundles combine multiple resources into one.
   This reduces the amount of server requests to be made by the web browser, and
   help with caching. This can be controlled with the ``bundle`` configuration

doc/paste_deploy.rst

 
   debug = True
 
+Use rolled up resources where possible and where they are available::
+
+  rollup = true
+
 Use bundling of resources::
 
   bundle = true

fanstatic/codegen.py

     dead[(resource.library, resource.relpath)] = True
     for depend in resource.depends:
         _visit(depend, result, dead)
+    for depend in resource.supersedes:
+        _visit(depend, result, dead)
     result.append(resource)
 
 
 def sort_resources_topological(resources):
-    """Sort resources by dependency.
+    """Sort resources by dependency and supersedes.
     """
     dead = {}
     result = []
             depends_s = ', depends=[%s]' % ', '.join(
                 [resource_to_name[(d.library, d.relpath)] for d in resource.depends])
             s += depends_s
+        if resource.supersedes:
+            supersedes_s = ', supersedes=[%s]' % ', '.join(
+                [resource_to_name[(i.library, i.relpath)] for i in resource.supersedes])
+            s += supersedes_s
         if resource.modes:
             items = []
             for mode_name, mode in resource.modes.items():

fanstatic/config.py

 from fanstatic import DEBUG, MINIFIED
 
 BOOL_CONFIG = set(['versioning', 'recompute_hashes', DEBUG, MINIFIED,
-                   'bottom', 'force_bottom', 'bundle', 'versioning_use_md5'])
+                   'bottom', 'force_bottom', 'bundle', 'rollup',
+                   'versioning_use_md5'])
 
 
 def convert_config(config):

fanstatic/core.py

       resources. If a string is given, a :py:class:`Resource` instance
       is constructed that has the same library as this resource.
 
+    :param supersedes: optionally, a list of :py:class:`Resource`
+      instances that this resource supersedes as a rollup
+      resource. If all these resources are required for render a page,
+      the superseding resource will be included instead.
+
     :param bottom: indicate that this resource is "bottom safe": it
       can be safely included on the bottom of the page (just before
       ``</body>``). This can be used to improve the performance of
 
     def __init__(self, library, relpath,
                  depends=None,
+                 supersedes=None,
                  bottom=False,
                  renderer=None,
                  debug=None,
                  dont_bundle=False,
-                 minified=None,
-                 supersedes=None, eager_superseder=None,
-                 ):
-        if supersedes is not None or eager_superseder is not None:
-            raise DeprecationWarning(
-                'Supersede functionality has been superseded by bundling')
+                 minified=None):
         self.library = library
         fullpath = os.path.join(library.path, relpath)
         if _resource_file_existence_checking and not os.path.exists(fullpath):
             minified.dependency_nr = self.dependency_nr
             minified.library_nr = self.library_nr
 
+
+        assert not isinstance(supersedes, basestring)
+        self.supersedes = supersedes or []
+
+        self.rollups = []
+        # create a reference to the superseder in the superseded resource
+        for resource in self.supersedes:
+            resource.rollups.append(self)
+        # also create a reference to the superseding mode in the superseded
+        # mode
+        # XXX what if mode is full-fledged resource which lists
+        # supersedes itself?
+        for mode_name, mode in self.modes.items():
+            for resource in self.supersedes:
+                superseded_mode = resource.mode(mode_name)
+                # if there is no such mode, let's skip it
+                if superseded_mode is resource:
+                    continue
+                mode.supersedes.append(superseded_mode)
+                superseded_mode.rollups.append(mode)
+
+
         # Register ourself with the Library.
         self.library.register(self)
 
       An exception is raised when both the ``debug`` and ``minified``
       parameters are ``True``.
 
+    :param rollup: If set to True (default is False) rolled up
+      combined resources will be served if they exist and supersede
+      existing resources that are needed.
+
     :param base_url: This URL will be prefixed in front of all resource
       URLs. This can be useful if your web framework wants the resources
       to be published on a sub-URL. Note that this can also be set
                  force_bottom=False,
                  minified=False,
                  debug=False,
+                 rollup=False,
                  base_url=None,
                  publisher_signature=DEFAULT_SIGNATURE,
                  bundle=False,
                  resources=None,
-                 rollup=None,
                  ):
-        if rollup is not None:
-            raise DeprecationWarning(
-                'Rollup has been superseded by bundling')
         self._versioning = versioning
         if versioning_use_md5:
             self._version_method = fanstatic.checksum.md5
         self._force_bottom = force_bottom
         self._base_url = base_url
         self._publisher_signature = publisher_signature
+        self._rollup = rollup
         self._bundle = bundle
         self._resources = set(resources or [])
         self._url_cache = {}  # prevent multiple computations per request
             resources.update(resource.resources)
 
         resources = [resource.mode(self._mode) for resource in resources]
+        if self._rollup:
+            resources = set(consolidate(resources))
         return sort_resources(resources)
 
     def clear(self):
     needed.clear()
 
 
+def consolidate(resources):
+    # keep track of rollups: rollup key -> set of resource keys
+    potential_rollups = {}
+    for resource in resources:
+        for rollup in resource.rollups:
+            s = potential_rollups.setdefault(
+                (rollup.library, rollup.relpath), set())
+            s.add((resource.library, resource.relpath))
+
+    # now go through resources, replacing them with rollups if
+    # conditions match
+    result = []
+    for resource in resources:
+        superseders = []
+        for rollup in resource.rollups:
+            s = potential_rollups[(rollup.library, rollup.relpath)]
+            if len(s) == len(rollup.supersedes):
+                superseders.append(rollup)
+        if superseders:
+            # use the exact superseder that rolls up the most
+            superseders = sorted(superseders, key=lambda i: len(i.supersedes))
+            result.append(superseders[-1])
+        else:
+            # nothing to supersede resource so use it directly
+            result.append(resource)
+    return result
+
 def sort_resources(resources):
     """Sort resources for inclusion on web page.
 

fanstatic/test_codegen.py

 i1 = Resource(foo, 'i1.js')
 i2 = Resource(foo, 'i2.js', depends=[i1])
 i3 = Resource(foo, 'i3.js', depends=[i2])
-i5 = Resource(foo, 'i5.js', depends=[i3])''' 
+i5 = Resource(foo, 'i5.js', depends=[i3])'''
+
+
+def test_generate_source_with_modes_and_rollup():
+    foo = Library('foo', '')
+    bar = Library('bar', '')
+    j1 = Resource(foo, 'j1.js', debug='j1-debug.js')
+    j2 = Resource(foo, 'j2.js', debug='j2-debug.js')
+    giantj = Resource(foo, 'giantj.js', supersedes=[j1, j2],
+                               debug='giantj-debug.js')
+    non_inlinable = Resource(foo, 'j3.js', debug=Resource(bar,
+                                                          'j4.js'))
+    assert generate_code(j1=j1, j2=j2, giantj=giantj,
+                         non_inlinable=non_inlinable) == '''\
+from fanstatic import Library, Resource
+
+# This code is auto-generated and not PEP8 compliant
+
+bar = Library('bar', '')
+foo = Library('foo', '')
+
+j1 = Resource(foo, 'j1.js', debug='j1-debug.js')
+j2 = Resource(foo, 'j2.js', debug='j2-debug.js')
+giantj = Resource(foo, 'giantj.js', supersedes=[j1, j2], debug='giantj-debug.js')
+non_inlinable = Resource(foo, 'j3.js', debug=Resource(bar, 'j4.js'))'''
+
 
 def test_generate_source_control_name():
     foo = Library('foo', '')

fanstatic/test_config.py

         'recompute_hashes': 'false',
         'bottom': 'True',
         'force_bottom': 'False',
+        'rollup': 0,
         'somethingelse': 'True',
         }
     assert convert_config(d) == {
         'recompute_hashes': False,
         'bottom': True,
         'force_bottom': False,
+        'rollup': False,
         'somethingelse': 'True',
         }
 
         'recompute_hashes': 'false',
         'bottom': 'True',
         'force_bottom': 'False',
+        'rollup': 0,
         }
     injector = make_injector(None, {}, **d)
     assert injector.app is None
         'recompute_hashes': False,
         'bottom': True,
         'force_bottom': False,
+        'rollup': False,
         }
 
 
         'recompute_hashes': 'false',
         'bottom': 'True',
         'force_bottom': 'False',
+        'rollup': 0,
         'publisher_signature': 'foo',
         }
     fanstatic = make_fanstatic(None, {}, **d)
         'recompute_hashes': False,
         'bottom': True,
         'force_bottom': False,
+        'rollup': False,
         'publisher_signature': 'foo',
         }

fanstatic/test_core.py

     assert more_stuff.resources == set([x1, x2, more_stuff])
 
 
-def test_deprecationwarnings():
-    with pytest.raises(DeprecationWarning):
-        NeededResources(rollup=True)
-
-    foo = Library('foo', '')
-    a1 = Resource(foo, 'a1.js')
-    a2 = Resource(foo, 'a2.js')
-    with pytest.raises(DeprecationWarning):
-        a3 = Resource(foo, 'a3.js', supersedes=[a1, a2])
-
-    with pytest.raises(DeprecationWarning):
-        a3 = Resource(foo, 'a3.js', eager_superseder=True)
-
-
 def test_convenience_need_not_initialized():
     foo = Library('foo', '')
     x1 = Resource(foo, 'a.js')
     l = Resource(foo, 'l.js', debug=l_debug, depends=[k])
     assert l_debug.dependency_nr == 1
 
-    
+
+def test_rollup():
+    foo = Library('foo', '')
+    b1 = Resource(foo, 'b1.js')
+    b2 = Resource(foo, 'b2.js')
+    giant = Resource(foo, 'giant.js', supersedes=[b1, b2])
+
+    needed = NeededResources(rollup=True)
+    needed.need(b1)
+    needed.need(b2)
+
+    assert needed.resources() == [giant]
+
+
+def test_rollup_cannot():
+    foo = Library('foo', '')
+    b1 = Resource(foo, 'b1.js')
+    b2 = Resource(foo, 'b2.js')
+
+    giant = Resource(foo, 'giant.js', supersedes=[b1, b2])
+
+    needed = NeededResources(rollup=True)
+    needed.need(b1)
+    assert needed.resources() == [b1]
+    assert giant not in needed.resources()
+
+
+def test_rollup_larger():
+    foo = Library('foo', '')
+    c1 = Resource(foo, 'c1.css')
+    c2 = Resource(foo, 'c2.css')
+    c3 = Resource(foo, 'c3.css')
+    giant = Resource(foo, 'giant.css', supersedes=[c1, c2, c3])
+
+    needed = NeededResources(rollup=True)
+    needed.need(c1)
+
+    assert needed.resources() == [c1]
+
+    needed.need(c2)
+
+    assert needed.resources() == [c1, c2]
+
+    needed.need(c3)
+
+    assert needed.resources() == [giant]
+
+
+def test_rollup_size_competing():
+    foo = Library('foo', '')
+    d1 = Resource(foo, 'd1.js')
+    d2 = Resource(foo, 'd2.js')
+    d3 = Resource(foo, 'd3.js')
+    giant = Resource(foo, 'giant.js', supersedes=[d1, d2])
+    giant_bigger = Resource(foo, 'giant-bigger.js',
+                            supersedes=[d1, d2, d3])
+
+    needed = NeededResources(rollup=True)
+    needed.need(d1)
+    needed.need(d2)
+    needed.need(d3)
+    assert needed.resources() == [giant_bigger]
+    assert giant not in needed.resources()
+
+
+def test_rollup_modes():
+    foo = Library('foo', '')
+    f1 = Resource(foo, 'f1.js', debug='f1-debug.js')
+    f2 = Resource(foo, 'f2.js', debug='f2-debug.js')
+    giantf = Resource(foo, 'giantf.js', supersedes=[f1, f2],
+                      debug='giantf-debug.js')
+
+    needed = NeededResources(rollup=True)
+    needed.need(f1)
+    needed.need(f2)
+    assert needed.resources() == [giantf]
+
+    needed = NeededResources(rollup=True, debug=True)
+    needed.need(f1)
+    needed.need(f2)
+    assert len(needed.resources()) == 1
+    assert needed.resources()[0].relpath == 'giantf-debug.js'
+
+
+def test_rollup_meaningless_rollup_mode():
+    foo = Library('foo', '')
+    g1 = Resource(foo, 'g1.js')
+    g2 = Resource(foo, 'g2.js')
+    giantg = Resource(foo, 'giantg.js', supersedes=[g1, g2],
+                      debug='giantg-debug.js')
+    needed = NeededResources(rollup=True)
+    needed.need(g1)
+    needed.need(g2)
+    assert needed.resources() == [giantg]
+
+    needed = NeededResources(rollup=True, debug=True)
+    needed.need(g1)
+    needed.need(g2)
+    assert needed.resources() == [giantg]
+
+
+def test_rollup_without_mode():
+    foo = Library('foo', '')
+    h1 = Resource(foo, 'h1.js', debug='h1-debug.js')
+    h2 = Resource(foo, 'h2.js', debug='h2-debug.js')
+    gianth = Resource(foo, 'gianth.js', supersedes=[h1, h2])
+
+    needed = NeededResources(rollup=True)
+    needed.need(h1)
+    needed.need(h2)
+    assert needed.resources() == [gianth]
+
+    needed = NeededResources(rollup=True, debug=True)
+    needed.need(h1)
+    needed.need(h2)
+    # no mode available for rollup
+    assert len(needed.resources()) == 2
+    assert needed.resources()[0].relpath == 'h1-debug.js'
+    assert needed.resources()[1].relpath == 'h2-debug.js'
+
 
 def test_rendering():
     foo = Library('foo', '')