Mercurial > hgsubversion
annotate tests/test_updatemeta.py @ 931:e1dbd9646d6a
svnwrap: use custom StringIO class in get_file()
The wrappers were calling ra.get_file() with a cStringIO object.
Empirically, svn 1.7.5 is writing 16kB blocks to the stream object, and
cStringIO reallocates its internal buffer and doubles its size whenever
it is filled. With large committed files this requires two large
memory blocks at the same time.
SimpleStringIO implements the mimimum StringIO interface used by
ra.get_file() but instead stores all the blocks and "join" them at the
end. It means more fragmentation but requires only one large block,
without overallocation. Also, 16kB blocks should be friendly to most
allocators.
In practice, this simple change let me convert a revision containing
multiple moderately large files, the largest being around 450MB, with a
32-bits Windows setup, python 2.7, swig svn 1.7.5, in stupid mode, while
it was previously aborting with "not enough memory". The same revision
still fails in replay mode.
author | Patrick Mezard <patrick@mezard.eu> |
---|---|
date | Sun, 16 Sep 2012 19:31:49 +0200 |
parents | 5bacb9c63e3e |
children |
rev | line source |
---|---|
922
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
1 import test_util |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
2 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
3 import os |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
4 import pickle |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
5 import unittest |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
6 import test_rebuildmeta |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
7 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
8 from mercurial import context |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
9 from mercurial import extensions |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
10 from mercurial import hg |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
11 from mercurial import ui |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
12 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
13 from hgsubversion import svncommands |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
14 from hgsubversion import svnmeta |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
15 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
16 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
17 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
18 def _do_case(self, name, stupid, single): |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
19 subdir = test_util.subdir.get(name, '') |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
20 layout = 'auto' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
21 if single: |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
22 layout = 'single' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
23 repo, repo_path = self.load_and_fetch(name, subdir=subdir, stupid=stupid, |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
24 layout=layout) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
25 assert len(self.repo) > 0 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
26 wc2_path = self.wc_path + '_clone' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
27 u = ui.ui() |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
28 src, dest = test_util.hgclone(u, self.wc_path, wc2_path, update=False) |
930
5bacb9c63e3e
Fix more peer breakage with old hg versions
Patrick Mezard <patrick@mezard.eu>
parents:
922
diff
changeset
|
29 src = test_util.getlocalpeer(src) |
5bacb9c63e3e
Fix more peer breakage with old hg versions
Patrick Mezard <patrick@mezard.eu>
parents:
922
diff
changeset
|
30 dest = test_util.getlocalpeer(dest) |
922
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
31 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
32 # insert a wrapper that prevents calling changectx.children() |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
33 def failfn(orig, ctx): |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
34 self.fail('calling %s is forbidden; it can cause massive slowdowns ' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
35 'when rebuilding large repositories' % orig) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
36 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
37 origchildren = getattr(context.changectx, 'children') |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
38 extensions.wrapfunction(context.changectx, 'children', failfn) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
39 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
40 # test updatemeta on an empty repo |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
41 try: |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
42 svncommands.updatemeta(u, dest, |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
43 args=[test_util.fileurl(repo_path + |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
44 subdir), ]) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
45 finally: |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
46 # remove the wrapper |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
47 context.changectx.children = origchildren |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
48 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
49 self._run_assertions(name, stupid, single, src, dest, u) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
50 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
51 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
52 def _run_assertions(self, name, stupid, single, src, dest, u): |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
53 test_rebuildmeta._run_assertions(self, name, stupid, single, src, dest, u) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
54 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
55 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
56 skip = set([ |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
57 'project_root_not_repo_root.svndump', |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
58 'corrupt.svndump', |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
59 ]) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
60 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
61 attrs = {'_do_case': _do_case, |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
62 '_run_assertions': _run_assertions, |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
63 } |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
64 for case in [f for f in os.listdir(test_util.FIXTURES) if f.endswith('.svndump')]: |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
65 # this fixture results in an empty repository, don't use it |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
66 if case in skip: |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
67 continue |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
68 bname = 'test_' + case[:-len('.svndump')] |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
69 attrs[bname] = test_rebuildmeta.buildmethod(case, bname, False, False) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
70 name = bname + '_stupid' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
71 attrs[name] = test_rebuildmeta.buildmethod(case, name, True, False) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
72 name = bname + '_single' |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
73 attrs[name] = test_rebuildmeta.buildmethod(case, name, False, True) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
74 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
75 UpdateMetaTests = type('UpdateMetaTests', (test_util.TestBase,), attrs) |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
76 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
77 |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
78 def suite(): |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
79 all_tests = [unittest.TestLoader().loadTestsFromTestCase(UpdateMetaTests), |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
80 ] |
6b7ac659c855
updatemeta: correctly handle empty metadata
Jun Fang <junfang@fb.com>
parents:
diff
changeset
|
81 return unittest.TestSuite(all_tests) |