Commits

Brodie Rao committed be5a9c7

Refreshing queue

  • Participants
  • Parent commits 390e511

Comments (0)

Files changed (7)

File bigger-revlog-cache

-# HG changeset patch
-# Parent de1d5c8a5d76e53fce52ee8664646ac2ed44017d
-revlog: increase chunk cache size from 64 KB to 128 KB
-
-diff --git a/mercurial/revlog.py b/mercurial/revlog.py
---- a/mercurial/revlog.py
-+++ b/mercurial/revlog.py
-@@ -820,9 +820,10 @@ class revlog(object):
-         else:
-             df = self.opener(self.datafile)
- 
--        realoffset = offset - offset % 65536
--        reallength = max(65536,
--                         65536 * int(math.ceil((offset + length) / 65536.0)))
-+        realoffset = offset - offset % 131072
-+        reallength = max(131072,
-+                         131072 * int(math.ceil((offset + length) /
-+                                                131072.0)))
-         df.seek(realoffset)
-         d = df.read(reallength)
-         df.close()
 # HG changeset patch
-# Parent da48091f55e19d9f89ce3756c32f66dc882c1a96
+# Parent 899e1589f17b78dcdaf39ac3000e3f4d339b1e7b
 localrepo: add store.cachefps config option to cache file handles
 
 When store.cachefps is set to True, all files opened from .hg/store stay open
          if expandpath:
              base = util.expandpath(base)
          if realpath:
-@@ -328,6 +332,12 @@ class vfs(abstractvfs):
+@@ -328,6 +332,9 @@ class vfs(abstractvfs):
              return
          os.chmod(name, self.createmode & 0666)
  
-+    def _invalidate(self, f):
-+        pass
-+
 +    def _open(self, f, mode):
 +        return util.posixfile(f, mode)
 +
      def __call__(self, path, mode="r", text=False, atomictemp=False):
          if self._audit:
              r = util.checkosfilename(path)
-@@ -350,6 +360,7 @@ class vfs(abstractvfs):
-                     return util.atomictempfile(f, mode, self.createmode)
-                 try:
-                     if 'w' in mode:
-+                        #self._invalidate(f)
-                         util.unlink(f)
-                         nlink = 0
-                     else:
-@@ -369,8 +380,9 @@ class vfs(abstractvfs):
-                     if self._trustnlink is None:
+@@ -370,7 +377,7 @@ class vfs(abstractvfs):
                          self._trustnlink = nlink > 1 or util.checknlink(f)
                      if nlink > 1 or not self._trustnlink:
-+                        #self._invalidate(f)
                          util.rename(util.mktempcopy(f), f)
 -        fp = util.posixfile(f, mode)
 +        fp = self._open(f, mode)
          if nlink == 0:
              self._fixfilemode(f)
          return fp
-@@ -400,6 +412,97 @@ class vfs(abstractvfs):
+@@ -400,6 +407,92 @@ class vfs(abstractvfs):
          else:
              return self.base
  
 +
 +        return fp
 +
-+    def _invalidate(self, f):
-+        for key in [k for k in self._fpcache if k[0] == f]:
-+            fp, st = self._fpcache.pop(key)
-+            fp._close()
-+
 +    def close(self):
 +        for entry in self._fpcache.itervalues():
 +            entry[0]._close()
  opener = vfs
  
  class auditvfs(object):
-@@ -424,6 +527,9 @@ class filtervfs(abstractvfs, auditvfs):
+@@ -424,6 +517,9 @@ class filtervfs(abstractvfs, auditvfs):
      def __call__(self, path, *args, **kwargs):
          return self.vfs(self._filter(path), *args, **kwargs)
  
      def join(self, path):
          if path:
              return self.vfs.join(self._filter(path))
-@@ -443,6 +549,8 @@ class readonlyvfs(abstractvfs, auditvfs)
+@@ -443,6 +539,8 @@ class readonlyvfs(abstractvfs, auditvfs)
              raise util.Abort('this vfs is read only')
          return self.vfs(path, mode, *args, **kw)
  

File chunked-revlog-cache

 # HG changeset patch
-# Parent c38c3fdc8b9317ba09e03ab09364c3800da7c50c
+# Parent cd18bc43b89a33f321b2025b4e7e32c8f33f4614
 revlog: read/cache chunks in fixed windows of 64 KB
 
+When reading a revlog chunk, instead of reading up to 64 KB ahead of the
+request offset and caching that, this change caches a fixed window before
+and after the requested data that falls on 64 KB boundaries. This increases
+cache hits when reading revlogs backwards.
+
+Running perfmoonwalk on the Mercurial repo (with almost 20,000 changesets) on
+Mac OS X with an SSD, before this change:
+
+$ hg perfmoonwalk
+! wall 2.307994 comb 2.310000 user 2.120000 sys 0.190000 (best of 5)
+
+(Each run has 10,668 cache hits and 9,304 misses.)
+
+After this change:
+
+$ hg perfmoonwalk
+! wall 1.814117 comb 1.810000 user 1.810000 sys 0.000000 (best of 6)
+
+(19,931 cache hits, 62 misses.)
+
 diff --git a/mercurial/revlog.py b/mercurial/revlog.py
 --- a/mercurial/revlog.py
 +++ b/mercurial/revlog.py
-@@ -15,7 +15,7 @@ and O(changes) merge between branches.
- from node import bin, hex, nullid, nullrev
- from i18n import _
- import ancestor, mdiff, parsers, error, util, templatefilters
--import struct, zlib, errno
-+import struct, zlib, errno, math
- 
- _pack = struct.pack
- _unpack = struct.unpack
-@@ -820,13 +820,15 @@ class revlog(object):
+@@ -820,13 +820,14 @@ class revlog(object):
          else:
              df = self.opener(self.datafile)
  
 -        readahead = max(65536, length)
 -        df.seek(offset)
 -        d = df.read(readahead)
-+        realoffset = offset - offset % 65536
-+        reallength = max(65536,
-+                         65536 * int(math.ceil((offset + length) / 65536.0)))
++        realoffset = offset & ~65535
++        reallength = ((offset + length + 65536) & ~65535) - realoffset
 +        df.seek(realoffset)
 +        d = df.read(reallength)
          df.close()

File configurable-revlog-cache

+# HG changeset patch
+# Parent bbe82e2081dce1cc67d1445b0d9204eed72ab878
+revlog: allow tuning of the chunk cache size (via format.chunkcachesize)
+
+Running perfmoonwalk on the Mercurial repo (with almost 20,000 changesets) on
+Mac OS X with an SSD, before this change:
+
+$ hg --config format.chunkcachesize=1024 perfmoonwalk
+! wall 2.022021 comb 2.030000 user 1.970000 sys 0.060000 (best of 5)
+
+(16,154 cache hits, 3,840 misses.)
+
+$ hg --config format.chunkcachesize=4096 perfmoonwalk
+! wall 1.901006 comb 1.900000 user 1.880000 sys 0.020000 (best of 6)
+
+(19,003 hits, 991 misses.)
+
+$ hg --config format.chunkcachesize=16384 perfmoonwalk
+! wall 1.802775 comb 1.800000 user 1.800000 sys 0.000000 (best of 6)
+
+(19,746 hits, 248 misses.)
+
+$ hg --config format.chunkcachesize=32768 perfmoonwalk
+! wall 1.818545 comb 1.810000 user 1.810000 sys 0.000000 (best of 6)
+
+(19,870 hits, 124 misses.)
+
+$ hg --config format.chunkcachesize=65536 perfmoonwalk
+! wall 1.801350 comb 1.810000 user 1.800000 sys 0.010000 (best of 6)
+
+(19,932 hits, 62 misses.)
+
+$ hg --config format.chunkcachesize=131072 perfmoonwalk
+! wall 1.805879 comb 1.820000 user 1.810000 sys 0.010000 (best of 6)
+
+(19,963 hits, 31 misses.)
+
+We may want to change the default size in the future based on testing and
+user feedback.
+
+diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
+--- a/mercurial/localrepo.py
++++ b/mercurial/localrepo.py
+@@ -279,6 +279,9 @@ class localrepository(object):
+         self.requirements = requirements
+         self.sopener.options = dict((r, 1) for r in requirements
+                                            if r in self.openerreqs)
++        chunkcachesize = self.ui.configint('format', 'chunkcachesize')
++        if chunkcachesize is not None:
++            self.sopener.options['chunkcachesize'] = chunkcachesize
+ 
+     def _writerequirements(self):
+         reqfile = self.opener("requires", "w")
+diff --git a/mercurial/revlog.py b/mercurial/revlog.py
+--- a/mercurial/revlog.py
++++ b/mercurial/revlog.py
+@@ -202,6 +202,7 @@ class revlog(object):
+         self._cache = None
+         self._basecache = None
+         self._chunkcache = (0, '')
++        self._chunkcachesize = 65536
+         self.index = []
+         self._pcache = {}
+         self._nodecache = {nullid: nullrev}
+@@ -215,6 +216,15 @@ class revlog(object):
+                     v |= REVLOGGENERALDELTA
+             else:
+                 v = 0
++            if 'chunkcachesize' in opts:
++                self._chunkcachesize = opts['chunkcachesize']
++
++        if self._chunkcachesize <= 0:
++            raise RevlogError(_('revlog chunk cache size %r is not greater '
++                                'than 0') % self._chunkcachesize)
++        elif self._chunkcachesize & (self._chunkcachesize - 1):
++            raise RevlogError(_('revlog chunk cache size %r is not a power '
++                                'of 2') % self._chunkcachesize)
+ 
+         i = ''
+         self._initempty = True
+@@ -820,8 +830,10 @@ class revlog(object):
+         else:
+             df = self.opener(self.datafile)
+ 
+-        realoffset = offset & ~65535
+-        reallength = ((offset + length + 65536) & ~65535) - realoffset
++        cachesize = self._chunkcachesize
++        realoffset = offset & ~(cachesize - 1)
++        reallength = (((offset + length + cachesize) & ~(cachesize - 1))
++                      - realoffset)
+         df.seek(realoffset)
+         d = df.read(reallength)
+         df.close()
+diff --git a/tests/test-init.t b/tests/test-init.t
+--- a/tests/test-init.t
++++ b/tests/test-init.t
+@@ -26,6 +26,31 @@ creating 'local'
+   $ hg ci --cwd local -A -m "init"
+   adding foo
+ 
++test custom revlog chunk cache sizes
++
++  $ hg --config format.chunkcachesize=0 log -R local -pv
++  abort: revlog chunk cache size 0 is not greater than 0!
++  [255]
++  $ hg --config format.chunkcachesize=1023 log -R local -pv
++  abort: revlog chunk cache size 1023 is not a power of 2!
++  [255]
++  $ hg --config format.chunkcachesize=1024 log -R local -pv
++  changeset:   0:08b9e9f63b32
++  tag:         tip
++  user:        test
++  date:        Thu Jan 01 00:00:00 1970 +0000
++  files:       foo
++  description:
++  init
++  
++  
++  diff -r 000000000000 -r 08b9e9f63b32 foo
++  --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
++  +++ b/foo	Thu Jan 01 00:00:00 1970 +0000
++  @@ -0,0 +1,1 @@
++  +this
++  
++
+ creating repo with format.usestore=false
+ 
+   $ hg --config format.usestore=false init old

File perfmoonwalk

+# HG changeset patch
+# Parent c38c3fdc8b9317ba09e03ab09364c3800da7c50c
+perf: add perfmoonwalk command to walk the changelog backwards
+
+This lets us test the effectiveness (or ineffectiveness) of the revlog chunk
+cache when walking revlogs backwards.
+
+diff --git a/contrib/perf.py b/contrib/perf.py
+--- a/contrib/perf.py
++++ b/contrib/perf.py
+@@ -243,6 +243,14 @@ def perflog(ui, repo, **opts):
+                                copies=opts.get('rename')))
+     ui.popbuffer()
+ 
++@command('perfmoonwalk')
++def perfmoonwalk(ui, repo):
++    def moonwalk():
++        for i in xrange(len(repo), -1, -1):
++            ctx = repo[i]
++            ctx.branch() # read changelog data (in addition to the index)
++    timer(moonwalk)
++
+ @command('perftemplating')
+ def perftemplating(ui, repo):
+     ui.pushbuffer()

File print-hit-miss

+# HG changeset patch
+# Parent b234fcc86319f8f5bfb501b0e89f7dccf059dc36
+diff --git a/mercurial/revlog.py b/mercurial/revlog.py
+--- a/mercurial/revlog.py
++++ b/mercurial/revlog.py
+@@ -839,9 +839,11 @@ class revlog(object):
+         cachestart = offset - o
+         cacheend = cachestart + length
+         if cachestart >= 0 and cacheend <= l:
++            print '\nhit'
+             if cachestart == 0 and cacheend == l:
+                 return d # avoid a copy
+             return util.buffer(d, cachestart, cacheend - cachestart)
++        print '\nmiss'
+ 
+         return self._loadchunk(offset, length)
+ 
+perfmoonwalk
+chunked-revlog-cache
+configurable-revlog-cache
+print-hit-miss
 fix-unshare
 cache-fps
-chunked-revlog-cache
-bigger-revlog-cache
 branch-cache-doc
 branches-reduce-reads
 branch-cache