changeset 944:d6db289f1548

pull: add hgsubversion.filestoresize to control memory consumption The configuration entry defines the size of the replay or stupid edited file store, that is the maximum amount of edited files data in megabytes which can be kept in memory before falling back to storing it in a temporary directory. Default to 200 (megabytes), use -1 to disable.
author Patrick Mezard <patrick@mezard.eu>
date Fri, 28 Sep 2012 21:43:50 +0200
parents c49c3c418f9d
children bfbfc9be3faa
files hgsubversion/editor.py hgsubversion/help/subversion.rst hgsubversion/stupid.py hgsubversion/util.py tests/test_fetch_renames.py tests/test_util.py
diffstat 6 files changed, 42 insertions(+), 7 deletions(-) [+]
line wrap: on
line diff
--- a/hgsubversion/editor.py
+++ b/hgsubversion/editor.py
@@ -95,7 +95,7 @@ class RevisionData(object):
         self.clear()
 
     def clear(self):
-        self.store = FileStore()
+        self.store = FileStore(util.getfilestoresize(self.ui))
         self.added = set()
         self.deleted = {}
         self.rev = None
--- a/hgsubversion/help/subversion.rst
+++ b/hgsubversion/help/subversion.rst
@@ -302,6 +302,25 @@ settings:
     be included or excluded. See the documentation for ``hg convert`` for more
     information on filemaps.
 
+  ``hgsubversion.filestoresize``
+
+    Maximum amount of temporary edited files data to be kept in memory,
+    in megabytes. The replay and stupid mode pull data by retrieving
+    delta information from the subversion repository and applying it on
+    known files data. Since the order of file edits is driven by the
+    subversion delta information order, edited files cannot be committed
+    immediately and are kept until all of them have been processed for
+    each changeset. ``filestoresize`` defines the maximum amount of
+    files data to be kept in memory before falling back to storing them
+    in a temporary directory. This setting is important with
+    repositories containing many files or large ones as both the
+    application of deltas and Mercurial commit process require the whole
+    file data to be available in memory. By limiting the amount of
+    temporary data kept in memory, larger files can be retrieved, at the
+    price of slower disk operations. Set it to a negative value to
+    disable the fallback behaviour and keep everything in memory.
+    Default to 200.
+
   ``hgsubversion.username``, ``hgsubversion.password``
 
     Set the username or password for accessing Subversion repositories.
--- a/hgsubversion/stupid.py
+++ b/hgsubversion/stupid.py
@@ -235,7 +235,7 @@ except AttributeError:
 def patchrepo(ui, meta, parentctx, patchfp):
     if not svnbackend:
         return patchrepoold(ui, meta, parentctx, patchfp)
-    store = patch.filestore()
+    store = patch.filestore(util.getfilestoresize(ui))
     try:
         touched = set()
         backend = svnbackend(ui, meta.repo, parentctx, store)
--- a/hgsubversion/util.py
+++ b/hgsubversion/util.py
@@ -327,3 +327,12 @@ revsets = {
     'fromsvn': revset_fromsvn,
     'svnrev': revset_svnrev,
 }
+
+def getfilestoresize(ui):
+    """Return the replay or stupid file memory store size in megabytes or -1"""
+    size = ui.configint('hgsubversion', 'filestoresize', 200)
+    if size >= 0:
+        size = size*(2**20)
+    else:
+        size = -1
+    return size
--- a/tests/test_fetch_renames.py
+++ b/tests/test_fetch_renames.py
@@ -14,8 +14,11 @@ class TestFetchRenames(test_util.TestBas
                 w('%s: %r %r\n' % (f, fctx.data(), fctx.renamed()))
 
     def _test_rename(self, stupid):
-        repo = self._load_fixture_and_fetch('renames.svndump', stupid=stupid)
-        # self._debug_print_copies(repo)
+        config = {
+            'hgsubversion.filestoresize': '0',
+            }
+        repo = self._load_fixture_and_fetch('renames.svndump', stupid=stupid,
+                config=config)
 
         # Map revnum to mappings of dest name to (source name, dest content)
         copies = {
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -333,8 +333,9 @@ class TestBase(unittest.TestCase):
             tarball.extract(entry, path)
         return path
 
-    def fetch(self, repo_path, subdir=None, stupid=False, layout='auto', startrev=0,
-              externals=None, noupdate=True, dest=None, rev=None):
+    def fetch(self, repo_path, subdir=None, stupid=False, layout='auto',
+            startrev=0, externals=None, noupdate=True, dest=None, rev=None,
+            config=None):
         if layout == 'single':
             if subdir is None:
                 subdir = 'trunk'
@@ -357,8 +358,11 @@ class TestBase(unittest.TestCase):
             cmd.append('--noupdate')
         if rev is not None:
             cmd.append('--rev=%s' % rev)
+        config = dict(config or {})
         if externals:
-            cmd[:0] = ['--config', 'hgsubversion.externals=%s' % externals]
+            config['hgsubversion.externals'] = str(externals)
+        for k,v in reversed(sorted(config.iteritems())):
+            cmd[:0] = ['--config', '%s=%s' % (k, v)]
 
         dispatch(cmd)