Commits

Anonymous committed 691078c

verify: use mercurials worker API to speedup stupid verify

We use mercurial internal worker api to distribute the workload
during verify. This helps with larger repositories.

with patch, intel i5 dualcore hyperthreading:

$ time hg svn verify
hg svn 169.47s user 21.27s system 373% cpu 51.050 total

before:

$ time hg svn verify
hg svn 100.48s user 11.85s system 99% cpu 1:52.97 total

This makes stupid verify as fast as subvertpy verify. We use \0 as a
separator for serializing the data as it's reserved on common fs.

Comments (0)

Files changed (1)

hgsubversion/verify.py

 
 from mercurial import util as hgutil
 from mercurial import error
+from mercurial import worker
 
 import svnwrap
 import svnrepo
 
         hgfiles = set(ctx) - util.ignoredfiles
 
+        def verifydata(svndata):
+            svnworker = svnrepo.svnremoterepo(ui, url).svn
+
+            i = 0
+            res = True
+            for fn, type in svndata:
+                i += 1
+                if type != 'f':
+                    continue
+
+                fp = fn
+                if branchpath:
+                    fp = branchpath + '/' + fn
+                data, mode = svnworker.get_file(posixpath.normpath(fp), srev)
+                try:
+                    fctx = ctx[fn]
+                except error.LookupError:
+                    yield i, "%s\0%r" % (fn, res)
+                    continue
+
+                if not fctx.data() == data:
+                    ui.write('difference in: %s\n' % fn)
+                    diff_file(fn, data)
+                    res = False
+                if not fctx.flags() == mode:
+                    ui.write('wrong flags for: %s\n' % fn)
+                    res = False
+                yield i, "%s\0%r" % (fn, res)
+
+        if url.startswith('file://'):
+            perarg = 0.00001
+        else:
+            perarg = 0.000001
+
         svndata = svn.list_files(branchpath, srev)
-        for i, (fn, type) in enumerate(svndata):
+        w = worker.worker(repo.ui, perarg, verifydata, (), tuple(svndata))
+        i = 0
+        for _, t in w:
             ui.progress('verify', i, total=len(hgfiles))
-
-            if type != 'f':
-                continue
+            i += 1
+            fn, ok = t.split('\0', 2)
+            if not bool(ok):
+                result = 1
             svnfiles.add(fn)
-            fp = fn
-            if branchpath:
-                fp = branchpath + '/' + fn
-            data, mode = svn.get_file(posixpath.normpath(fp), srev)
-            try:
-                fctx = ctx[fn]
-            except error.LookupError:
-                result = 1
-                continue
-            if not fctx.data() == data:
-                ui.write('difference in: %s\n' % fn)
-                diff_file(fn, data)
-                result = 1
-            if not fctx.flags() == mode:
-                ui.write('wrong flags for: %s\n' % fn)
-                result = 1
 
         if hgfiles != svnfiles:
             unexpected = hgfiles - svnfiles