# HG changeset patch # User Dan Villiom Podlaski Christiansen # Date 1465283753 -7200 # Node ID 019c3e194fba3d920a0c56f801c406bae8511481 # Parent 77da55e0baa4aab8e61af1f7312e2645b895afa8 tests: optimise creating repositories and loading dumps Previously, we'd use svnadmin for creating repositories and loading dumps. That tends to be a bit slow, as it forks a new process and loads the Subversion libraries into it. Instead, we extend our existing Subversion wrappers and load the dumps using the API. This is a noticable speedup. The only downside is that we rely on Subversion and Subvertpy to correctly close all file descriptors; an assumption which hasn't always held in the past. I ran some benchmarks on my relatively slow Mac with $TMPDIR on a ramdisk, and they showed a significant change: I compared ten runs of each with Subvertpy: min: -18.8% (299.1s -> 243.0s) median: -20.0% (307.1s -> 245.6s) ...and three runs of each with SWIG: min: -22.8% (368.7s -> 284.7s) median: -25.7% (384.4s -> 285.5s) (Since the timing measures wall clock time, the minimum time is likely to be the most accurate and useful measurement.) diff --git a/hgsubversion/svnwrap/subvertpy_wrapper.py b/hgsubversion/svnwrap/subvertpy_wrapper.py --- a/hgsubversion/svnwrap/subvertpy_wrapper.py +++ b/hgsubversion/svnwrap/subvertpy_wrapper.py @@ -22,6 +22,7 @@ try: from subvertpy import delta from subvertpy import properties from subvertpy import ra + from subvertpy import repos import subvertpy except ImportError: raise ImportError('Subvertpy %d.%d.%d or later required, but not found' @@ -51,6 +52,18 @@ def version(): svnvers += '-' + subversion_version[3] return (svnvers, 'Subvertpy ' + _versionstr(subvertpy.__version__)) +def create_and_load(repopath, dumpfd): + ''' create a new repository at repopath and load the given dump into it ''' + repo = repos.create(repopath) + + nullfd = open(os.devnull, 'w') + + try: + repo.load_fs(dumpfd, nullfd, repos.LOAD_UUID_FORCE) + finally: + dumpfd.close() + nullfd.close() + # exported values ERR_FS_ALREADY_EXISTS = subvertpy.ERR_FS_ALREADY_EXISTS ERR_FS_CONFLICT = subvertpy.ERR_FS_CONFLICT diff --git a/hgsubversion/svnwrap/svn_swig_wrapper.py b/hgsubversion/svnwrap/svn_swig_wrapper.py --- a/hgsubversion/svnwrap/svn_swig_wrapper.py +++ b/hgsubversion/svnwrap/svn_swig_wrapper.py @@ -21,6 +21,7 @@ try: from svn import core from svn import delta from svn import ra + from svn import repos subversion_version = (core.SVN_VER_MAJOR, core.SVN_VER_MINOR, core.SVN_VER_MICRO) @@ -36,6 +37,21 @@ if subversion_version < required_binding def version(): return '%d.%d.%d' % subversion_version, 'SWIG' +def create_and_load(repopath, dumpfd): + ''' create a new repository at repopath and load the given dump into it ''' + pool = core.Pool() + r = repos.svn_repos_create(repopath, '', '', None, None, pool) + + try: + repos.svn_repos_load_fs2(r, dumpfd, None, + repos.svn_repos_load_uuid_force, + '', False, False, None, pool) + finally: + dumpfd.close() + + pool.destroy() + + # exported values ERR_FS_ALREADY_EXISTS = core.SVN_ERR_FS_ALREADY_EXISTS ERR_FS_CONFLICT = core.SVN_ERR_FS_CONFLICT diff --git a/tests/test_svnwrap.py b/tests/test_svnwrap.py --- a/tests/test_svnwrap.py +++ b/tests/test_svnwrap.py @@ -11,15 +11,11 @@ class TestBasicRepoLayout(unittest.TestC def setUp(self): self.tmpdir = tempfile.mkdtemp('svnwrap_test') self.repo_path = '%s/testrepo' % self.tmpdir - subprocess.call(['svnadmin', 'create', self.repo_path, ]) - inp = open(os.path.join(os.path.dirname(__file__), 'fixtures', - 'project_root_at_repo_root.svndump')) - proc = subprocess.call(['svnadmin', 'load', self.repo_path, ], - stdin=inp, - close_fds=test_util.canCloseFds, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - assert proc == 0 + + with open(os.path.join(test_util.FIXTURES, + 'project_root_at_repo_root.svndump')) as fp: + svnwrap.create_and_load(self.repo_path, fp) + self.repo = svnwrap.SubversionRepo(test_util.fileurl(self.repo_path)) def tearDown(self): diff --git a/tests/test_util.py b/tests/test_util.py --- a/tests/test_util.py +++ b/tests/test_util.py @@ -47,6 +47,7 @@ except AttributeError: except ImportError: SkipTest = None +from hgsubversion import svnwrap from hgsubversion import util from hgsubversion import svnwrap @@ -534,12 +535,8 @@ class TestBase(unittest.TestCase): ''' path = self._makerepopath() assert not os.path.exists(path) - subprocess.call(['svnadmin', 'create', path,], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - inp = open(os.path.join(FIXTURES, fixture_name)) - proc = subprocess.Popen(['svnadmin', 'load', path,], stdin=inp, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - proc.communicate() + with open(os.path.join(FIXTURES, fixture_name)) as inp: + svnwrap.create_and_load(path, inp) return path def load_repo_tarball(self, fixture_name): @@ -596,7 +593,7 @@ class TestBase(unittest.TestCase): return hg.repository(testui(), self.wc_path) - def load_and_fetch(self, fixture_name, *args, **opts): + def load(self, fixture_name): if fixture_name.endswith('.svndump'): repo_path = self.load_svndump(fixture_name) elif fixture_name.endswith('tar.gz'): @@ -604,6 +601,10 @@ class TestBase(unittest.TestCase): else: assert False, 'Unknown fixture type' + return repo_path + + def load_and_fetch(self, fixture_name, *args, **opts): + repo_path = self.load(fixture_name) return self.fetch(repo_path, *args, **opts), repo_path def _load_fixture_and_fetch(self, *args, **kwargs):