annotate tests/test_updatemeta.py @ 931:e1dbd9646d6a

svnwrap: use custom StringIO class in get_file() The wrappers were calling ra.get_file() with a cStringIO object. Empirically, svn 1.7.5 is writing 16kB blocks to the stream object, and cStringIO reallocates its internal buffer and doubles its size whenever it is filled. With large committed files this requires two large memory blocks at the same time. SimpleStringIO implements the mimimum StringIO interface used by ra.get_file() but instead stores all the blocks and "join" them at the end. It means more fragmentation but requires only one large block, without overallocation. Also, 16kB blocks should be friendly to most allocators. In practice, this simple change let me convert a revision containing multiple moderately large files, the largest being around 450MB, with a 32-bits Windows setup, python 2.7, swig svn 1.7.5, in stupid mode, while it was previously aborting with "not enough memory". The same revision still fails in replay mode.
author Patrick Mezard <patrick@mezard.eu>
date Sun, 16 Sep 2012 19:31:49 +0200
parents 5bacb9c63e3e
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
922
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
1 import test_util
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
2
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
3 import os
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
4 import pickle
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
5 import unittest
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
6 import test_rebuildmeta
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
7
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
8 from mercurial import context
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
9 from mercurial import extensions
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
10 from mercurial import hg
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
11 from mercurial import ui
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
12
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
13 from hgsubversion import svncommands
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
14 from hgsubversion import svnmeta
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
15
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
16
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
17
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
18 def _do_case(self, name, stupid, single):
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
19 subdir = test_util.subdir.get(name, '')
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
20 layout = 'auto'
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
21 if single:
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
22 layout = 'single'
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
23 repo, repo_path = self.load_and_fetch(name, subdir=subdir, stupid=stupid,
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
24 layout=layout)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
25 assert len(self.repo) > 0
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
26 wc2_path = self.wc_path + '_clone'
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
27 u = ui.ui()
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
28 src, dest = test_util.hgclone(u, self.wc_path, wc2_path, update=False)
930
5bacb9c63e3e Fix more peer breakage with old hg versions
Patrick Mezard <patrick@mezard.eu>
parents: 922
diff changeset
29 src = test_util.getlocalpeer(src)
5bacb9c63e3e Fix more peer breakage with old hg versions
Patrick Mezard <patrick@mezard.eu>
parents: 922
diff changeset
30 dest = test_util.getlocalpeer(dest)
922
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
31
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
32 # insert a wrapper that prevents calling changectx.children()
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
33 def failfn(orig, ctx):
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
34 self.fail('calling %s is forbidden; it can cause massive slowdowns '
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
35 'when rebuilding large repositories' % orig)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
36
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
37 origchildren = getattr(context.changectx, 'children')
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
38 extensions.wrapfunction(context.changectx, 'children', failfn)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
39
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
40 # test updatemeta on an empty repo
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
41 try:
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
42 svncommands.updatemeta(u, dest,
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
43 args=[test_util.fileurl(repo_path +
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
44 subdir), ])
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
45 finally:
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
46 # remove the wrapper
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
47 context.changectx.children = origchildren
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
48
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
49 self._run_assertions(name, stupid, single, src, dest, u)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
50
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
51
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
52 def _run_assertions(self, name, stupid, single, src, dest, u):
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
53 test_rebuildmeta._run_assertions(self, name, stupid, single, src, dest, u)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
54
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
55
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
56 skip = set([
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
57 'project_root_not_repo_root.svndump',
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
58 'corrupt.svndump',
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
59 ])
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
60
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
61 attrs = {'_do_case': _do_case,
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
62 '_run_assertions': _run_assertions,
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
63 }
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
64 for case in [f for f in os.listdir(test_util.FIXTURES) if f.endswith('.svndump')]:
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
65 # this fixture results in an empty repository, don't use it
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
66 if case in skip:
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
67 continue
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
68 bname = 'test_' + case[:-len('.svndump')]
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
69 attrs[bname] = test_rebuildmeta.buildmethod(case, bname, False, False)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
70 name = bname + '_stupid'
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
71 attrs[name] = test_rebuildmeta.buildmethod(case, name, True, False)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
72 name = bname + '_single'
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
73 attrs[name] = test_rebuildmeta.buildmethod(case, name, False, True)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
74
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
75 UpdateMetaTests = type('UpdateMetaTests', (test_util.TestBase,), attrs)
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
76
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
77
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
78 def suite():
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
79 all_tests = [unittest.TestLoader().loadTestsFromTestCase(UpdateMetaTests),
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
80 ]
6b7ac659c855 updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff changeset
81 return unittest.TestSuite(all_tests)