Mercurial > hgsubversion
changeset 1457:019c3e194fba
tests: optimise creating repositories and loading dumps
Previously, we'd use svnadmin for creating repositories and loading
dumps. That tends to be a bit slow, as it forks a new process and
loads the Subversion libraries into it. Instead, we extend our
existing Subversion wrappers and load the dumps using the API.
This is a noticable speedup. The only downside is that we rely on
Subversion and Subvertpy to correctly close all file descriptors; an
assumption which hasn't always held in the past.
I ran some benchmarks on my relatively slow Mac with $TMPDIR on a
ramdisk, and they showed a significant change:
I compared ten runs of each with Subvertpy:
min: -18.8% (299.1s -> 243.0s)
median: -20.0% (307.1s -> 245.6s)
...and three runs of each with SWIG:
min: -22.8% (368.7s -> 284.7s)
median: -25.7% (384.4s -> 285.5s)
(Since the timing measures wall clock time, the minimum time is likely
to be the most accurate and useful measurement.)
author | Dan Villiom Podlaski Christiansen <danchr@gmail.com> |
---|---|
date | Tue, 07 Jun 2016 09:15:53 +0200 |
parents | 77da55e0baa4 |
children | dcf9eff9b5b7 |
files | hgsubversion/svnwrap/subvertpy_wrapper.py hgsubversion/svnwrap/svn_swig_wrapper.py tests/test_svnwrap.py tests/test_util.py |
diffstat | 4 files changed, 42 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/hgsubversion/svnwrap/subvertpy_wrapper.py +++ b/hgsubversion/svnwrap/subvertpy_wrapper.py @@ -22,6 +22,7 @@ try: from subvertpy import delta from subvertpy import properties from subvertpy import ra + from subvertpy import repos import subvertpy except ImportError: raise ImportError('Subvertpy %d.%d.%d or later required, but not found' @@ -51,6 +52,18 @@ def version(): svnvers += '-' + subversion_version[3] return (svnvers, 'Subvertpy ' + _versionstr(subvertpy.__version__)) +def create_and_load(repopath, dumpfd): + ''' create a new repository at repopath and load the given dump into it ''' + repo = repos.create(repopath) + + nullfd = open(os.devnull, 'w') + + try: + repo.load_fs(dumpfd, nullfd, repos.LOAD_UUID_FORCE) + finally: + dumpfd.close() + nullfd.close() + # exported values ERR_FS_ALREADY_EXISTS = subvertpy.ERR_FS_ALREADY_EXISTS ERR_FS_CONFLICT = subvertpy.ERR_FS_CONFLICT
--- a/hgsubversion/svnwrap/svn_swig_wrapper.py +++ b/hgsubversion/svnwrap/svn_swig_wrapper.py @@ -21,6 +21,7 @@ try: from svn import core from svn import delta from svn import ra + from svn import repos subversion_version = (core.SVN_VER_MAJOR, core.SVN_VER_MINOR, core.SVN_VER_MICRO) @@ -36,6 +37,21 @@ if subversion_version < required_binding def version(): return '%d.%d.%d' % subversion_version, 'SWIG' +def create_and_load(repopath, dumpfd): + ''' create a new repository at repopath and load the given dump into it ''' + pool = core.Pool() + r = repos.svn_repos_create(repopath, '', '', None, None, pool) + + try: + repos.svn_repos_load_fs2(r, dumpfd, None, + repos.svn_repos_load_uuid_force, + '', False, False, None, pool) + finally: + dumpfd.close() + + pool.destroy() + + # exported values ERR_FS_ALREADY_EXISTS = core.SVN_ERR_FS_ALREADY_EXISTS ERR_FS_CONFLICT = core.SVN_ERR_FS_CONFLICT
--- a/tests/test_svnwrap.py +++ b/tests/test_svnwrap.py @@ -11,15 +11,11 @@ class TestBasicRepoLayout(unittest.TestC def setUp(self): self.tmpdir = tempfile.mkdtemp('svnwrap_test') self.repo_path = '%s/testrepo' % self.tmpdir - subprocess.call(['svnadmin', 'create', self.repo_path, ]) - inp = open(os.path.join(os.path.dirname(__file__), 'fixtures', - 'project_root_at_repo_root.svndump')) - proc = subprocess.call(['svnadmin', 'load', self.repo_path, ], - stdin=inp, - close_fds=test_util.canCloseFds, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - assert proc == 0 + + with open(os.path.join(test_util.FIXTURES, + 'project_root_at_repo_root.svndump')) as fp: + svnwrap.create_and_load(self.repo_path, fp) + self.repo = svnwrap.SubversionRepo(test_util.fileurl(self.repo_path)) def tearDown(self):
--- a/tests/test_util.py +++ b/tests/test_util.py @@ -47,6 +47,7 @@ except AttributeError: except ImportError: SkipTest = None +from hgsubversion import svnwrap from hgsubversion import util from hgsubversion import svnwrap @@ -534,12 +535,8 @@ class TestBase(unittest.TestCase): ''' path = self._makerepopath() assert not os.path.exists(path) - subprocess.call(['svnadmin', 'create', path,], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - inp = open(os.path.join(FIXTURES, fixture_name)) - proc = subprocess.Popen(['svnadmin', 'load', path,], stdin=inp, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - proc.communicate() + with open(os.path.join(FIXTURES, fixture_name)) as inp: + svnwrap.create_and_load(path, inp) return path def load_repo_tarball(self, fixture_name): @@ -596,7 +593,7 @@ class TestBase(unittest.TestCase): return hg.repository(testui(), self.wc_path) - def load_and_fetch(self, fixture_name, *args, **opts): + def load(self, fixture_name): if fixture_name.endswith('.svndump'): repo_path = self.load_svndump(fixture_name) elif fixture_name.endswith('tar.gz'): @@ -604,6 +601,10 @@ class TestBase(unittest.TestCase): else: assert False, 'Unknown fixture type' + return repo_path + + def load_and_fetch(self, fixture_name, *args, **opts): + repo_path = self.load(fixture_name) return self.fetch(repo_path, *args, **opts), repo_path def _load_fixture_and_fetch(self, *args, **kwargs):