# HG changeset patch # User Patrick Mezard # Date 1230933245 21600 # Node ID f80132c5fea574f3ec63e79062231023ecb69cd3 # Parent f244eaee506999863fcdea45ac8db3034db89ca6 Convert svn:externals properties into a .hgsvnexternals file diff --git a/fetch_command.py b/fetch_command.py --- a/fetch_command.py +++ b/fetch_command.py @@ -12,6 +12,7 @@ from svn import delta import hg_delta_editor import svnwrap +import svnexternals import util @@ -425,6 +426,55 @@ def getcopies(svn, hg_editor, branch, br hgcopies.update(copies) return hgcopies +def stupid_fetch_externals(svn, branchpath, r, parentctx): + """Extract svn:externals for the current revision and branch + + Return an externalsfile instance or None if there are no externals + to convert and never were. + """ + externals = svnexternals.externalsfile() + if '.hgsvnexternals' in parentctx: + externals.read(parentctx['.hgsvnexternals'].data()) + # Detect property additions only, changes are handled by checking + # existing entries individually. Projects are unlikely to store + # externals on many different root directories, so we trade code + # duplication and complexity for a constant lookup price at every + # revision in the common case. + dirs = set(externals) + if parentctx.node() == revlog.nullid: + dirs.update([p for p,k in svn.list_files(branchpath, r.revnum) if k == 'd']) + dirs.add('') + else: + branchprefix = branchpath + '/' + for path, e in r.paths.iteritems(): + if e.action == 'D': + continue + if not path.startswith(branchprefix) and path != branchpath: + continue + kind = svn.checkpath(path, r.revnum) + if kind != 'd': + continue + path = path[len(branchprefix):] + dirs.add(path) + if e.action == 'M': + continue + for child, k in svn.list_files(branchprefix + path, r.revnum): + if k == 'd': + dirs.add((path + '/' + child).strip('/')) + + # Retrieve new or updated values + for dir in dirs: + try: + values = svn.list_props(branchpath + '/' + dir, r.revnum) + externals[dir] = values.get('svn:externals', '') + except IOError: + externals[dir] = '' + + if not externals and '.hgsvnexternals' not in parentctx: + # Do not create empty externals files + return None + return externals + def stupid_fetch_branchrev(svn, hg_editor, branch, branchpath, r, parentctx): """Extract all 'branch' content at a given revision. @@ -449,7 +499,6 @@ def stupid_fetch_branchrev(svn, hg_edito files.append(path) elif kind == 'd': if e.action == 'M': - # Ignore property changes for now continue dirpath = branchprefix + path for child, k in svn.list_files(dirpath, r.revnum): @@ -495,14 +544,24 @@ def stupid_svn_server_pull_rev(ui, svn, continue else: try: - files_touched, filectxfn = stupid_diff_branchrev( + files_touched, filectxfn2 = stupid_diff_branchrev( ui, svn, hg_editor, b, r, parentctx) except BadPatchApply, e: # Either this revision or the previous one does not exist. ui.status("fetching entire rev: %s.\n" % e.message) - files_touched, filectxfn = stupid_fetch_branchrev( + files_touched, filectxfn2 = stupid_fetch_branchrev( svn, hg_editor, b, branches[b], r, parentctx) + externals = stupid_fetch_externals(svn, branches[b], r, parentctx) + if externals is not None: + files_touched.append('.hgsvnexternals') + + def filectxfn(repo, memctx, path): + if path == '.hgsvnexternals': + return context.memfilectx(path=path, data=externals.write(), + islink=False, isexec=False, copied=None) + return filectxfn2(repo, memctx, path) + extra = util.build_extra(r.revnum, b, svn.uuid, svn.subdir) if '' in files_touched: files_touched.remove('') diff --git a/hg_delta_editor.py b/hg_delta_editor.py --- a/hg_delta_editor.py +++ b/hg_delta_editor.py @@ -14,6 +14,7 @@ from mercurial import node from svn import delta from svn import core +import svnexternals import util as our_util def pickle_atomic(data, file_path, dir=None): @@ -144,12 +145,14 @@ class HgChangeReceiver(delta.Editor): self.current_rev = None self.current_files_exec = {} self.current_files_symlink = {} + self.dir_batons = {} # Map fully qualified destination file paths to module source path self.copies = {} self.missing_plaintexts = set() self.commit_branches_empty = {} self.base_revision = None self.branches_to_delete = set() + self.externals = {} def _save_metadata(self): '''Save the Subversion metadata. This should really be called after @@ -349,12 +352,41 @@ class HgChangeReceiver(delta.Editor): self.branches.update(added_branches) self._save_metadata() + def _updateexternals(self): + if not self.externals: + return + # Accumulate externals records for all branches + revnum = self.current_rev.revnum + branches = {} + for path, entry in self.externals.iteritems(): + if not self._is_path_valid(path): + continue + p, b, bp = self._split_branch_path(path) + if bp not in branches: + external = svnexternals.externalsfile() + parent = self.get_parent_revision(revnum, b) + pctx = self.repo[parent] + if '.hgsvnexternals' in pctx: + external.read(pctx['.hgsvnexternals'].data()) + branches[bp] = external + else: + external = branches[bp] + external[p] = entry + + # Register the file changes + for bp, external in branches.iteritems(): + path = bp + '/.hgsvnexternals' + self.current_files[path] = external.write() + self.current_files_symlink[path] = False + self.current_files_exec[path] = False + def commit_current_delta(self): if hasattr(self, '_exception_info'): #pragma: no cover traceback.print_exception(*self._exception_info) raise ReplayException() if self.missing_plaintexts: raise MissingPlainTextError() + self._updateexternals() files_to_commit = self.current_files.keys() files_to_commit.extend(self.current_files_symlink.keys()) files_to_commit.extend(self.current_files_exec.keys()) @@ -420,6 +452,11 @@ class HgChangeReceiver(delta.Editor): and branch not in self.repo.branchtags()): continue parent_ctx = self.repo.changectx(parents[0]) + if '.hgsvnexternals' not in parent_ctx and '.hgsvnexternals' in files: + # Do not register empty externals files + if not self.current_files[files['.hgsvnexternals']]: + del files['.hgsvnexternals'] + def filectxfn(repo, memctx, path): current_file = files[path] if current_file in self.deleted_files: @@ -579,6 +616,7 @@ class HgChangeReceiver(delta.Editor): if br_path != '': br_path2 = br_path + '/' # assuming it is a directory + self.externals[path] = None def delete_x(x): self.deleted_files[x] = True map(delete_x, [pat for pat in self.current_files.iterkeys() @@ -666,6 +704,7 @@ class HgChangeReceiver(delta.Editor): @stash_exception_on_self def add_directory(self, path, parent_baton, copyfrom_path, copyfrom_revision, dir_pool=None): + self.dir_batons[path] = path br_path, branch = self._path_and_branch_for_path(path) if br_path is not None: if not copyfrom_path and not br_path: @@ -673,14 +712,14 @@ class HgChangeReceiver(delta.Editor): else: self.commit_branches_empty[branch] = False if br_path is None or not copyfrom_path: - return + return path if copyfrom_path: tag = self._is_path_tag(copyfrom_path) if tag not in self.tags: tag = None if not self._is_path_valid(copyfrom_path) and not tag: self.missing_plaintexts.add('%s/' % path) - return + return path if tag: source_branch, source_rev = self.tags[tag] @@ -692,7 +731,7 @@ class HgChangeReceiver(delta.Editor): source_branch) if new_hash == node.nullid: self.missing_plaintexts.add('%s/' % path) - return + return path cp_f_ctx = self.repo.changectx(new_hash) if cp_f != '/' and cp_f != '': cp_f = '%s/' % cp_f @@ -718,6 +757,7 @@ class HgChangeReceiver(delta.Editor): parentctx = self.repo.changectx(parentid) if self.aresamefiles(parentctx, cp_f_ctx, copies.values()): self.copies.update(copies) + return path @stash_exception_on_self def change_file_prop(self, file_baton, name, value, pool=None): @@ -726,11 +766,26 @@ class HgChangeReceiver(delta.Editor): elif name == 'svn:special': self.current_files_symlink[self.current_file] = bool(value is not None) + @stash_exception_on_self + def change_dir_prop(self, dir_baton, name, value, pool=None): + if dir_baton is None: + return + path = self.dir_batons[dir_baton] + if name == 'svn:externals': + self.externals[path] = value + @stash_exception_on_self def open_directory(self, path, parent_baton, base_revision, dir_pool=None): + self.dir_batons[path] = path p_, branch = self._path_and_branch_for_path(path) if p_ == '': self.commit_branches_empty[branch] = False + return path + + @stash_exception_on_self + def close_directory(self, dir_baton, dir_pool=None): + if dir_baton is not None: + del self.dir_batons[dir_baton] @stash_exception_on_self def apply_textdelta(self, file_baton, base_checksum, pool=None): diff --git a/svnexternals.py b/svnexternals.py new file mode 100644 --- /dev/null +++ b/svnexternals.py @@ -0,0 +1,59 @@ +import cStringIO + +from mercurial import util as merc_util + +class externalsfile(dict): + """Map svn directories to lists of externals entries. + """ + def __init__(self): + super(externalsfile, self).__init__() + self.encoding = 'utf-8' + + def __setitem__(self, key, value): + if value is None: + value = [] + elif isinstance(value, basestring): + value = value.splitlines() + if key == '.': + key = '' + if not value: + if key in self: + del self[key] + else: + super(externalsfile, self).__setitem__(key, value) + + def write(self): + fp = cStringIO.StringIO() + for target in merc_util.sort(self): + lines = self[target] + if not lines: + continue + if not target: + target = '.' + fp.write('[%s]\n' % target) + for l in lines: + l = ' ' + l + '\n' + fp.write(l) + return fp.getvalue() + + def read(self, data): + self.clear() + fp = cStringIO.StringIO(data) + dirs = {} + target = None + for line in fp.readlines(): + if not line.strip(): + continue + if line.startswith('['): + line = line.strip() + if line[-1] != ']': + raise merc_util.Abort('invalid externals section name: %s' % line) + target = line[1:-1] + if target == '.': + target = '' + elif line.startswith(' '): + line = line.rstrip('\n') + if target is None or not line: + continue + self.setdefault(target, []).append(line[1:]) + diff --git a/tests/fixtures/externals.sh b/tests/fixtures/externals.sh new file mode 100755 --- /dev/null +++ b/tests/fixtures/externals.sh @@ -0,0 +1,66 @@ +#!/bin/sh +# +# Generate externals.svndump +# + +mkdir temp +cd temp + +mkdir project-orig +cd project-orig +mkdir trunk +mkdir branches +mkdir externals +cd .. + +svnadmin create testrepo +svnurl=file://`pwd`/testrepo +svn import project-orig $svnurl -m "init project" + +svn co $svnurl project +cd project/externals +mkdir project1 +echo a > project1/a +svn add project1 +mkdir project2 +echo a > project2/b +svn add project2 +svn ci -m "configure externals projects" +cd ../trunk +# Add an external reference +echo a > a +svn add a +cat > externals < externals < externals < externals < ../externals.svndump diff --git a/tests/fixtures/externals.svndump b/tests/fixtures/externals.svndump new file mode 100644 --- /dev/null +++ b/tests/fixtures/externals.svndump @@ -0,0 +1,305 @@ +SVN-fs-dump-format-version: 2 + +UUID: a5e66397-f826-4e1c-bc26-5123bb064477 + +Revision-number: 0 +Prop-content-length: 56 +Content-length: 56 + +K 8 +svn:date +V 27 +2008-12-27T12:49:06.601814Z +PROPS-END + +Revision-number: 1 +Prop-content-length: 114 +Content-length: 114 + +K 7 +svn:log +V 12 +init project +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:06.652695Z +PROPS-END + +Node-path: branches +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Node-path: externals +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Node-path: trunk +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Revision-number: 2 +Prop-content-length: 130 +Content-length: 130 + +K 7 +svn:log +V 28 +configure externals projects +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:07.236694Z +PROPS-END + +Node-path: externals/project1 +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Node-path: externals/project1/a +Node-kind: file +Node-action: add +Prop-content-length: 10 +Text-content-length: 2 +Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3 +Content-length: 12 + +PROPS-END +a + + +Node-path: externals/project2 +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Node-path: externals/project2/b +Node-kind: file +Node-action: add +Prop-content-length: 10 +Text-content-length: 2 +Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3 +Content-length: 12 + +PROPS-END +a + + +Revision-number: 3 +Prop-content-length: 120 +Content-length: 120 + +K 7 +svn:log +V 18 +set externals on . +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:08.250996Z +PROPS-END + +Node-path: trunk +Node-kind: dir +Node-action: change +Prop-content-length: 71 +Content-length: 71 + +K 13 +svn:externals +V 36 +../externals/project1 deps/project1 + +PROPS-END + + +Node-path: trunk/a +Node-kind: file +Node-action: add +Prop-content-length: 10 +Text-content-length: 2 +Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3 +Content-length: 12 + +PROPS-END +a + + +Revision-number: 4 +Prop-content-length: 123 +Content-length: 123 + +K 7 +svn:log +V 21 +update externals on . +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:09.190834Z +PROPS-END + +Node-path: trunk +Node-kind: dir +Node-action: change +Prop-content-length: 107 +Content-length: 107 + +K 13 +svn:externals +V 72 +../externals/project1 deps/project1 +../externals/project2 deps/project2 + +PROPS-END + + +Revision-number: 5 +Prop-content-length: 115 +Content-length: 115 + +K 7 +svn:log +V 13 +add on subdir +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:10.289216Z +PROPS-END + +Node-path: trunk +Node-kind: dir +Node-action: change +Prop-content-length: 71 +Content-length: 71 + +K 13 +svn:externals +V 36 +../externals/project2 deps/project2 + +PROPS-END + + +Node-path: trunk/subdir +Node-kind: dir +Node-action: add +Prop-content-length: 71 +Content-length: 71 + +K 13 +svn:externals +V 36 +../externals/project1 deps/project1 + +PROPS-END + + +Node-path: trunk/subdir2 +Node-kind: dir +Node-action: add +Prop-content-length: 71 +Content-length: 71 + +K 13 +svn:externals +V 36 +../externals/project1 deps/project1 + +PROPS-END + + +Revision-number: 6 +Prop-content-length: 125 +Content-length: 125 + +K 7 +svn:log +V 23 +remove externals subdir +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:11.184732Z +PROPS-END + +Node-path: trunk/subdir +Node-action: delete + + +Revision-number: 7 +Prop-content-length: 126 +Content-length: 126 + +K 7 +svn:log +V 24 +remove externals subdir2 +K 10 +svn:author +V 7 +pmezard +K 8 +svn:date +V 27 +2008-12-27T12:49:12.194114Z +PROPS-END + +Node-path: trunk/subdir2 +Node-kind: dir +Node-action: change +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + diff --git a/tests/run.py b/tests/run.py --- a/tests/run.py +++ b/tests/run.py @@ -6,6 +6,7 @@ sys.path.append(os.path.dirname(os.path. import test_binaryfiles import test_diff +import test_externals import test_fetch_branches import test_fetch_command import test_fetch_command_regexes @@ -25,6 +26,7 @@ import test_utility_commands def suite(): return unittest.TestSuite([test_binaryfiles.suite(), test_diff.suite(), + test_externals.suite(), test_fetch_branches.suite(), test_fetch_command.suite(), test_fetch_command_regexes.suite(), diff --git a/tests/test_externals.py b/tests/test_externals.py new file mode 100644 --- /dev/null +++ b/tests/test_externals.py @@ -0,0 +1,87 @@ +import cStringIO +import os +import shutil +import sys +import tempfile +import unittest + +from mercurial import hg +from mercurial import ui +from mercurial import node + +import fetch_command +import svnexternals +import test_util + + +class TestFetchExternals(test_util.TestBase): + def test_externalsfile(self): + f = svnexternals.externalsfile() + f['t1'] = 'dir1 -r10 svn://foobar' + f['t 2'] = 'dir2 -r10 svn://foobar' + f['t3'] = ['dir31 -r10 svn://foobar', 'dir32 -r10 svn://foobar'] + + refext = """\ +[t 2] + dir2 -r10 svn://foobar +[t1] + dir1 -r10 svn://foobar +[t3] + dir31 -r10 svn://foobar + dir32 -r10 svn://foobar +""" + value = f.write() + self.assertEqual(refext, value) + + f2 = svnexternals.externalsfile() + f2.read(value) + self.assertEqual(sorted(f), sorted(f2)) + for t in f: + self.assertEqual(f[t], f2[t]) + + def test_externals(self, stupid=False): + repo = self._load_fixture_and_fetch('externals.svndump', stupid=stupid) + + ref0 = """\ +[.] + ../externals/project1 deps/project1 +""" + self.assertEqual(ref0, repo[0]['.hgsvnexternals'].data()) + ref1 = """\ +[.] + ../externals/project1 deps/project1 + ../externals/project2 deps/project2 +""" + self.assertEqual(ref1, repo[1]['.hgsvnexternals'].data()) + + ref2 = """\ +[.] + ../externals/project2 deps/project2 +[subdir] + ../externals/project1 deps/project1 +[subdir2] + ../externals/project1 deps/project1 +""" + self.assertEqual(ref2, repo[2]['.hgsvnexternals'].data()) + + ref3 = """\ +[.] + ../externals/project2 deps/project2 +[subdir2] + ../externals/project1 deps/project1 +""" + self.assertEqual(ref3, repo[3]['.hgsvnexternals'].data()) + + ref4 = """\ +[.] + ../externals/project2 deps/project2 +""" + self.assertEqual(ref4, repo[4]['.hgsvnexternals'].data()) + + def test_externals_stupid(self): + self.test_externals(True) + +def suite(): + all = [unittest.TestLoader().loadTestsFromTestCase(TestFetchExternals), + ] + return unittest.TestSuite(all)