Commits

Patrick Mézard committed d6db289

pull: add hgsubversion.filestoresize to control memory consumption

The configuration entry defines the size of the replay or stupid edited
file store, that is the maximum amount of edited files data in megabytes
which can be kept in memory before falling back to storing it in a
temporary directory. Default to 200 (megabytes), use -1 to disable.

Comments (0)

Files changed (6)

hgsubversion/editor.py

         self.clear()
 
     def clear(self):
-        self.store = FileStore()
+        self.store = FileStore(util.getfilestoresize(self.ui))
         self.added = set()
         self.deleted = {}
         self.rev = None

hgsubversion/help/subversion.rst

     be included or excluded. See the documentation for ``hg convert`` for more
     information on filemaps.
 
+  ``hgsubversion.filestoresize``
+
+    Maximum amount of temporary edited files data to be kept in memory,
+    in megabytes. The replay and stupid mode pull data by retrieving
+    delta information from the subversion repository and applying it on
+    known files data. Since the order of file edits is driven by the
+    subversion delta information order, edited files cannot be committed
+    immediately and are kept until all of them have been processed for
+    each changeset. ``filestoresize`` defines the maximum amount of
+    files data to be kept in memory before falling back to storing them
+    in a temporary directory. This setting is important with
+    repositories containing many files or large ones as both the
+    application of deltas and Mercurial commit process require the whole
+    file data to be available in memory. By limiting the amount of
+    temporary data kept in memory, larger files can be retrieved, at the
+    price of slower disk operations. Set it to a negative value to
+    disable the fallback behaviour and keep everything in memory.
+    Default to 200.
+
   ``hgsubversion.username``, ``hgsubversion.password``
 
     Set the username or password for accessing Subversion repositories.

hgsubversion/stupid.py

 def patchrepo(ui, meta, parentctx, patchfp):
     if not svnbackend:
         return patchrepoold(ui, meta, parentctx, patchfp)
-    store = patch.filestore()
+    store = patch.filestore(util.getfilestoresize(ui))
     try:
         touched = set()
         backend = svnbackend(ui, meta.repo, parentctx, store)

hgsubversion/util.py

     'fromsvn': revset_fromsvn,
     'svnrev': revset_svnrev,
 }
+
+def getfilestoresize(ui):
+    """Return the replay or stupid file memory store size in megabytes or -1"""
+    size = ui.configint('hgsubversion', 'filestoresize', 200)
+    if size >= 0:
+        size = size*(2**20)
+    else:
+        size = -1
+    return size

tests/test_fetch_renames.py

                 w('%s: %r %r\n' % (f, fctx.data(), fctx.renamed()))
 
     def _test_rename(self, stupid):
-        repo = self._load_fixture_and_fetch('renames.svndump', stupid=stupid)
-        # self._debug_print_copies(repo)
+        config = {
+            'hgsubversion.filestoresize': '0',
+            }
+        repo = self._load_fixture_and_fetch('renames.svndump', stupid=stupid,
+                config=config)
 
         # Map revnum to mappings of dest name to (source name, dest content)
         copies = {

tests/test_util.py

             tarball.extract(entry, path)
         return path
 
-    def fetch(self, repo_path, subdir=None, stupid=False, layout='auto', startrev=0,
-              externals=None, noupdate=True, dest=None, rev=None):
+    def fetch(self, repo_path, subdir=None, stupid=False, layout='auto',
+            startrev=0, externals=None, noupdate=True, dest=None, rev=None,
+            config=None):
         if layout == 'single':
             if subdir is None:
                 subdir = 'trunk'
             cmd.append('--noupdate')
         if rev is not None:
             cmd.append('--rev=%s' % rev)
+        config = dict(config or {})
         if externals:
-            cmd[:0] = ['--config', 'hgsubversion.externals=%s' % externals]
+            config['hgsubversion.externals'] = str(externals)
+        for k,v in reversed(sorted(config.iteritems())):
+            cmd[:0] = ['--config', '%s=%s' % (k, v)]
 
         dispatch(cmd)