changeset 1457:019c3e194fba

tests: optimise creating repositories and loading dumps Previously, we'd use svnadmin for creating repositories and loading dumps. That tends to be a bit slow, as it forks a new process and loads the Subversion libraries into it. Instead, we extend our existing Subversion wrappers and load the dumps using the API. This is a noticable speedup. The only downside is that we rely on Subversion and Subvertpy to correctly close all file descriptors; an assumption which hasn't always held in the past. I ran some benchmarks on my relatively slow Mac with $TMPDIR on a ramdisk, and they showed a significant change: I compared ten runs of each with Subvertpy: min: -18.8% (299.1s -> 243.0s) median: -20.0% (307.1s -> 245.6s) ...and three runs of each with SWIG: min: -22.8% (368.7s -> 284.7s) median: -25.7% (384.4s -> 285.5s) (Since the timing measures wall clock time, the minimum time is likely to be the most accurate and useful measurement.)
author Dan Villiom Podlaski Christiansen <danchr@gmail.com>
date Tue, 07 Jun 2016 09:15:53 +0200
parents 77da55e0baa4
children dcf9eff9b5b7
files hgsubversion/svnwrap/subvertpy_wrapper.py hgsubversion/svnwrap/svn_swig_wrapper.py tests/test_svnwrap.py tests/test_util.py
diffstat 4 files changed, 42 insertions(+), 16 deletions(-) [+]
line wrap: on
line diff
--- a/hgsubversion/svnwrap/subvertpy_wrapper.py
+++ b/hgsubversion/svnwrap/subvertpy_wrapper.py
@@ -22,6 +22,7 @@ try:
     from subvertpy import delta
     from subvertpy import properties
     from subvertpy import ra
+    from subvertpy import repos
     import subvertpy
 except ImportError:
     raise ImportError('Subvertpy %d.%d.%d or later required, but not found'
@@ -51,6 +52,18 @@ def version():
         svnvers += '-' + subversion_version[3]
     return (svnvers, 'Subvertpy ' + _versionstr(subvertpy.__version__))
 
+def create_and_load(repopath, dumpfd):
+    ''' create a new repository at repopath and load the given dump into it '''
+    repo = repos.create(repopath)
+
+    nullfd = open(os.devnull, 'w')
+
+    try:
+        repo.load_fs(dumpfd, nullfd, repos.LOAD_UUID_FORCE)
+    finally:
+        dumpfd.close()
+        nullfd.close()
+
 # exported values
 ERR_FS_ALREADY_EXISTS = subvertpy.ERR_FS_ALREADY_EXISTS
 ERR_FS_CONFLICT = subvertpy.ERR_FS_CONFLICT
--- a/hgsubversion/svnwrap/svn_swig_wrapper.py
+++ b/hgsubversion/svnwrap/svn_swig_wrapper.py
@@ -21,6 +21,7 @@ try:
     from svn import core
     from svn import delta
     from svn import ra
+    from svn import repos
 
     subversion_version = (core.SVN_VER_MAJOR, core.SVN_VER_MINOR,
                           core.SVN_VER_MICRO)
@@ -36,6 +37,21 @@ if subversion_version < required_binding
 def version():
     return '%d.%d.%d' % subversion_version, 'SWIG'
 
+def create_and_load(repopath, dumpfd):
+    ''' create a new repository at repopath and load the given dump into it '''
+    pool = core.Pool()
+    r = repos.svn_repos_create(repopath, '', '', None, None, pool)
+
+    try:
+        repos.svn_repos_load_fs2(r, dumpfd, None,
+                                 repos.svn_repos_load_uuid_force,
+                                 '', False, False, None, pool)
+    finally:
+        dumpfd.close()
+
+        pool.destroy()
+
+
 # exported values
 ERR_FS_ALREADY_EXISTS = core.SVN_ERR_FS_ALREADY_EXISTS
 ERR_FS_CONFLICT = core.SVN_ERR_FS_CONFLICT
--- a/tests/test_svnwrap.py
+++ b/tests/test_svnwrap.py
@@ -11,15 +11,11 @@ class TestBasicRepoLayout(unittest.TestC
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp('svnwrap_test')
         self.repo_path = '%s/testrepo' % self.tmpdir
-        subprocess.call(['svnadmin', 'create', self.repo_path, ])
-        inp = open(os.path.join(os.path.dirname(__file__), 'fixtures',
-                                'project_root_at_repo_root.svndump'))
-        proc = subprocess.call(['svnadmin', 'load', self.repo_path, ],
-                                stdin=inp,
-                                close_fds=test_util.canCloseFds,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
-        assert proc == 0
+
+        with open(os.path.join(test_util.FIXTURES,
+                               'project_root_at_repo_root.svndump')) as fp:
+            svnwrap.create_and_load(self.repo_path, fp)
+
         self.repo = svnwrap.SubversionRepo(test_util.fileurl(self.repo_path))
 
     def tearDown(self):
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -47,6 +47,7 @@ except AttributeError:
         except ImportError:
             SkipTest = None
 
+from hgsubversion import svnwrap
 from hgsubversion import util
 from hgsubversion import svnwrap
 
@@ -534,12 +535,8 @@ class TestBase(unittest.TestCase):
         '''
         path = self._makerepopath()
         assert not os.path.exists(path)
-        subprocess.call(['svnadmin', 'create', path,],
-                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        inp = open(os.path.join(FIXTURES, fixture_name))
-        proc = subprocess.Popen(['svnadmin', 'load', path,], stdin=inp,
-                                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        proc.communicate()
+        with open(os.path.join(FIXTURES, fixture_name)) as inp:
+            svnwrap.create_and_load(path, inp)
         return path
 
     def load_repo_tarball(self, fixture_name):
@@ -596,7 +593,7 @@ class TestBase(unittest.TestCase):
 
         return hg.repository(testui(), self.wc_path)
 
-    def load_and_fetch(self, fixture_name, *args, **opts):
+    def load(self, fixture_name):
         if fixture_name.endswith('.svndump'):
             repo_path = self.load_svndump(fixture_name)
         elif fixture_name.endswith('tar.gz'):
@@ -604,6 +601,10 @@ class TestBase(unittest.TestCase):
         else:
             assert False, 'Unknown fixture type'
 
+        return repo_path
+
+    def load_and_fetch(self, fixture_name, *args, **opts):
+        repo_path = self.load(fixture_name)
         return self.fetch(repo_path, *args, **opts), repo_path
 
     def _load_fixture_and_fetch(self, *args, **kwargs):