changeset 174:f80132c5fea5

Convert svn:externals properties into a .hgsvnexternals file
author Patrick Mezard <pmezard@gmail.com>
date Fri, 02 Jan 2009 15:54:05 -0600
parents f244eaee5069
children 2412800b1258
files fetch_command.py hg_delta_editor.py svnexternals.py tests/fixtures/externals.sh tests/fixtures/externals.svndump tests/run.py tests/test_externals.py
diffstat 7 files changed, 639 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/fetch_command.py
+++ b/fetch_command.py
@@ -12,6 +12,7 @@ from svn import delta
 
 import hg_delta_editor
 import svnwrap
+import svnexternals
 import util
 
 
@@ -425,6 +426,55 @@ def getcopies(svn, hg_editor, branch, br
         hgcopies.update(copies)
     return hgcopies
 
+def stupid_fetch_externals(svn, branchpath, r, parentctx):
+    """Extract svn:externals for the current revision and branch
+
+    Return an externalsfile instance or None if there are no externals
+    to convert and never were.
+    """
+    externals = svnexternals.externalsfile()
+    if '.hgsvnexternals' in parentctx:
+        externals.read(parentctx['.hgsvnexternals'].data())
+    # Detect property additions only, changes are handled by checking
+    # existing entries individually. Projects are unlikely to store
+    # externals on many different root directories, so we trade code
+    # duplication and complexity for a constant lookup price at every
+    # revision in the common case.
+    dirs = set(externals)
+    if parentctx.node() == revlog.nullid:
+        dirs.update([p for p,k in svn.list_files(branchpath, r.revnum) if k == 'd'])
+        dirs.add('')
+    else:
+        branchprefix = branchpath + '/'
+        for path, e in r.paths.iteritems():
+            if e.action == 'D':
+                continue
+            if not path.startswith(branchprefix) and path != branchpath:
+                continue
+            kind = svn.checkpath(path, r.revnum)
+            if kind != 'd':
+                continue
+            path = path[len(branchprefix):]
+            dirs.add(path)
+            if e.action == 'M':
+                continue
+            for child, k in svn.list_files(branchprefix + path, r.revnum):
+                if k == 'd':
+                    dirs.add((path + '/' + child).strip('/'))
+
+    # Retrieve new or updated values
+    for dir in dirs:
+        try:
+            values = svn.list_props(branchpath + '/' + dir, r.revnum)
+            externals[dir] = values.get('svn:externals', '')
+        except IOError:
+            externals[dir] = ''
+
+    if not externals and '.hgsvnexternals' not in parentctx:
+        # Do not create empty externals files
+        return None
+    return externals
+
 def stupid_fetch_branchrev(svn, hg_editor, branch, branchpath, r, parentctx):
     """Extract all 'branch' content at a given revision.
 
@@ -449,7 +499,6 @@ def stupid_fetch_branchrev(svn, hg_edito
                 files.append(path)
             elif kind == 'd':
                 if e.action == 'M':
-                    # Ignore property changes for now
                     continue
                 dirpath = branchprefix + path
                 for child, k in svn.list_files(dirpath, r.revnum):
@@ -495,14 +544,24 @@ def stupid_svn_server_pull_rev(ui, svn, 
             continue
         else:
             try:
-                files_touched, filectxfn = stupid_diff_branchrev(
+                files_touched, filectxfn2 = stupid_diff_branchrev(
                     ui, svn, hg_editor, b, r, parentctx)
             except BadPatchApply, e:
                 # Either this revision or the previous one does not exist.
                 ui.status("fetching entire rev: %s.\n" % e.message)
-                files_touched, filectxfn = stupid_fetch_branchrev(
+                files_touched, filectxfn2 = stupid_fetch_branchrev(
                     svn, hg_editor, b, branches[b], r, parentctx)
 
+            externals = stupid_fetch_externals(svn, branches[b], r, parentctx)
+            if externals is not None:
+                files_touched.append('.hgsvnexternals')
+
+            def filectxfn(repo, memctx, path):
+                if path == '.hgsvnexternals':
+                    return context.memfilectx(path=path, data=externals.write(), 
+                                              islink=False, isexec=False, copied=None)
+                return filectxfn2(repo, memctx, path)
+            
         extra = util.build_extra(r.revnum, b, svn.uuid, svn.subdir)
         if '' in files_touched:
             files_touched.remove('')
--- a/hg_delta_editor.py
+++ b/hg_delta_editor.py
@@ -14,6 +14,7 @@ from mercurial import node
 from svn import delta
 from svn import core
 
+import svnexternals
 import util as our_util
 
 def pickle_atomic(data, file_path, dir=None):
@@ -144,12 +145,14 @@ class HgChangeReceiver(delta.Editor):
         self.current_rev = None
         self.current_files_exec = {}
         self.current_files_symlink = {}
+        self.dir_batons = {}
         # Map fully qualified destination file paths to module source path
         self.copies = {}
         self.missing_plaintexts = set()
         self.commit_branches_empty = {}
         self.base_revision = None
         self.branches_to_delete = set()
+        self.externals = {}
 
     def _save_metadata(self):
         '''Save the Subversion metadata. This should really be called after
@@ -349,12 +352,41 @@ class HgChangeReceiver(delta.Editor):
         self.branches.update(added_branches)
         self._save_metadata()
 
+    def _updateexternals(self):
+        if not self.externals:
+            return
+        # Accumulate externals records for all branches
+        revnum = self.current_rev.revnum
+        branches = {}
+        for path, entry in self.externals.iteritems():
+            if not self._is_path_valid(path):
+                continue
+            p, b, bp = self._split_branch_path(path)
+            if bp not in branches:
+                external = svnexternals.externalsfile()
+                parent = self.get_parent_revision(revnum, b)
+                pctx = self.repo[parent]
+                if '.hgsvnexternals' in pctx:
+                    external.read(pctx['.hgsvnexternals'].data())
+                branches[bp] = external
+            else:
+                external = branches[bp]
+            external[p] = entry
+
+        # Register the file changes
+        for bp, external in branches.iteritems():
+            path = bp + '/.hgsvnexternals'
+            self.current_files[path] = external.write()
+            self.current_files_symlink[path] = False
+            self.current_files_exec[path] = False
+
     def commit_current_delta(self):
         if hasattr(self, '_exception_info'):  #pragma: no cover
             traceback.print_exception(*self._exception_info)
             raise ReplayException()
         if self.missing_plaintexts:
             raise MissingPlainTextError()
+        self._updateexternals()
         files_to_commit = self.current_files.keys()
         files_to_commit.extend(self.current_files_symlink.keys())
         files_to_commit.extend(self.current_files_exec.keys())
@@ -420,6 +452,11 @@ class HgChangeReceiver(delta.Editor):
                     and branch not in self.repo.branchtags()):
                     continue
             parent_ctx = self.repo.changectx(parents[0])
+            if '.hgsvnexternals' not in parent_ctx and '.hgsvnexternals' in files:
+                # Do not register empty externals files
+                if not self.current_files[files['.hgsvnexternals']]:
+                    del files['.hgsvnexternals']
+
             def filectxfn(repo, memctx, path):
                 current_file = files[path]
                 if current_file in self.deleted_files:
@@ -579,6 +616,7 @@ class HgChangeReceiver(delta.Editor):
                 if br_path != '':
                     br_path2 = br_path + '/'
                 # assuming it is a directory
+                self.externals[path] = None
                 def delete_x(x):
                     self.deleted_files[x] = True
                 map(delete_x, [pat for pat in self.current_files.iterkeys()
@@ -666,6 +704,7 @@ class HgChangeReceiver(delta.Editor):
     @stash_exception_on_self
     def add_directory(self, path, parent_baton, copyfrom_path,
                       copyfrom_revision, dir_pool=None):
+        self.dir_batons[path] = path
         br_path, branch = self._path_and_branch_for_path(path)
         if br_path is not None:
             if not copyfrom_path and not br_path:
@@ -673,14 +712,14 @@ class HgChangeReceiver(delta.Editor):
             else:
                 self.commit_branches_empty[branch] = False
         if br_path is None or not copyfrom_path:
-            return
+            return path
         if copyfrom_path:
             tag = self._is_path_tag(copyfrom_path)
             if tag not in self.tags:
                 tag = None
             if not self._is_path_valid(copyfrom_path) and not tag:
                 self.missing_plaintexts.add('%s/' % path)
-                return
+                return path
 
         if tag:
             source_branch, source_rev = self.tags[tag]
@@ -692,7 +731,7 @@ class HgChangeReceiver(delta.Editor):
                                             source_branch)
         if new_hash == node.nullid:
             self.missing_plaintexts.add('%s/' % path)
-            return
+            return path
         cp_f_ctx = self.repo.changectx(new_hash)
         if cp_f != '/' and cp_f != '':
             cp_f = '%s/' % cp_f
@@ -718,6 +757,7 @@ class HgChangeReceiver(delta.Editor):
                 parentctx = self.repo.changectx(parentid)
                 if self.aresamefiles(parentctx, cp_f_ctx, copies.values()):
                     self.copies.update(copies)
+        return path
 
     @stash_exception_on_self
     def change_file_prop(self, file_baton, name, value, pool=None):
@@ -726,11 +766,26 @@ class HgChangeReceiver(delta.Editor):
         elif name == 'svn:special':
             self.current_files_symlink[self.current_file] = bool(value is not None)
 
+    @stash_exception_on_self
+    def change_dir_prop(self, dir_baton, name, value, pool=None):
+        if dir_baton is None:
+            return
+        path = self.dir_batons[dir_baton]
+        if name == 'svn:externals':
+            self.externals[path] = value
+
     @stash_exception_on_self
     def open_directory(self, path, parent_baton, base_revision, dir_pool=None):
+        self.dir_batons[path] = path
         p_, branch = self._path_and_branch_for_path(path)
         if p_ == '':
             self.commit_branches_empty[branch] = False
+        return path
+
+    @stash_exception_on_self
+    def close_directory(self, dir_baton, dir_pool=None):
+        if dir_baton is not None:
+            del self.dir_batons[dir_baton]
 
     @stash_exception_on_self
     def apply_textdelta(self, file_baton, base_checksum, pool=None):
new file mode 100644
--- /dev/null
+++ b/svnexternals.py
@@ -0,0 +1,59 @@
+import cStringIO
+
+from mercurial import util as merc_util
+
+class externalsfile(dict):
+    """Map svn directories to lists of externals entries.
+    """
+    def __init__(self):
+        super(externalsfile, self).__init__()
+        self.encoding = 'utf-8'
+
+    def __setitem__(self, key, value):
+        if value is None:
+            value = []
+        elif isinstance(value, basestring):
+            value = value.splitlines()
+        if key == '.':
+            key = ''
+        if not value:
+            if key in self:
+                del self[key]
+        else:
+            super(externalsfile, self).__setitem__(key, value)
+
+    def write(self):
+        fp = cStringIO.StringIO()
+        for target in merc_util.sort(self):
+            lines = self[target]
+            if not lines:
+                continue
+            if not target:
+                target = '.'
+            fp.write('[%s]\n' % target)
+            for l in lines:
+                l = ' ' + l + '\n'
+                fp.write(l)
+        return fp.getvalue()
+
+    def read(self, data):
+        self.clear()
+        fp = cStringIO.StringIO(data)
+        dirs = {}
+        target = None
+        for line in fp.readlines():
+            if not line.strip():
+                continue
+            if line.startswith('['):
+                line = line.strip()
+                if line[-1] != ']':
+                    raise merc_util.Abort('invalid externals section name: %s' % line)
+                target = line[1:-1]
+                if target == '.':
+                    target = ''
+            elif line.startswith(' '):
+                line = line.rstrip('\n')
+                if target is None or not line:
+                    continue
+                self.setdefault(target, []).append(line[1:])
+            
new file mode 100755
--- /dev/null
+++ b/tests/fixtures/externals.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# Generate externals.svndump
+#
+
+mkdir temp
+cd temp
+
+mkdir project-orig
+cd project-orig
+mkdir trunk
+mkdir branches
+mkdir externals
+cd ..
+
+svnadmin create testrepo
+svnurl=file://`pwd`/testrepo
+svn import project-orig $svnurl -m "init project"
+
+svn co $svnurl project
+cd project/externals
+mkdir project1
+echo a > project1/a
+svn add project1
+mkdir project2
+echo a > project2/b
+svn add project2
+svn ci -m "configure externals projects"
+cd ../trunk
+# Add an external reference
+echo a > a
+svn add a
+cat > externals <<EOF
+../externals/project1 deps/project1
+EOF
+svn propset -F externals svn:externals .
+svn ci -m "set externals on ."
+# Add another one
+cat > externals <<EOF
+../externals/project1 deps/project1
+../externals/project2 deps/project2
+EOF
+svn propset -F externals svn:externals .
+svn ci -m "update externals on ."
+# Suppress an external and add one on a subdir
+cat > externals <<EOF
+../externals/project2 deps/project2
+EOF
+svn propset -F externals svn:externals .
+mkdir subdir
+mkdir subdir2
+svn add subdir subdir2
+cat > externals <<EOF
+../externals/project1 deps/project1
+EOF
+svn propset -F externals svn:externals subdir subdir2
+svn ci -m "add on subdir"
+# Suppress the subdirectory
+svn rm subdir
+svn ci -m 'remove externals subdir'
+# Remove the property on subdir2
+svn propdel svn:externals subdir2
+svn ci -m 'remove externals subdir2'
+cd ../..
+
+svnadmin dump testrepo > ../externals.svndump
new file mode 100644
--- /dev/null
+++ b/tests/fixtures/externals.svndump
@@ -0,0 +1,305 @@
+SVN-fs-dump-format-version: 2
+
+UUID: a5e66397-f826-4e1c-bc26-5123bb064477
+
+Revision-number: 0
+Prop-content-length: 56
+Content-length: 56
+
+K 8
+svn:date
+V 27
+2008-12-27T12:49:06.601814Z
+PROPS-END
+
+Revision-number: 1
+Prop-content-length: 114
+Content-length: 114
+
+K 7
+svn:log
+V 12
+init project
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:06.652695Z
+PROPS-END
+
+Node-path: branches
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: externals
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: trunk
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Revision-number: 2
+Prop-content-length: 130
+Content-length: 130
+
+K 7
+svn:log
+V 28
+configure externals projects
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:07.236694Z
+PROPS-END
+
+Node-path: externals/project1
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: externals/project1/a
+Node-kind: file
+Node-action: add
+Prop-content-length: 10
+Text-content-length: 2
+Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3
+Content-length: 12
+
+PROPS-END
+a
+
+
+Node-path: externals/project2
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: externals/project2/b
+Node-kind: file
+Node-action: add
+Prop-content-length: 10
+Text-content-length: 2
+Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3
+Content-length: 12
+
+PROPS-END
+a
+
+
+Revision-number: 3
+Prop-content-length: 120
+Content-length: 120
+
+K 7
+svn:log
+V 18
+set externals on .
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:08.250996Z
+PROPS-END
+
+Node-path: trunk
+Node-kind: dir
+Node-action: change
+Prop-content-length: 71
+Content-length: 71
+
+K 13
+svn:externals
+V 36
+../externals/project1 deps/project1
+
+PROPS-END
+
+
+Node-path: trunk/a
+Node-kind: file
+Node-action: add
+Prop-content-length: 10
+Text-content-length: 2
+Text-content-md5: 60b725f10c9c85c70d97880dfe8191b3
+Content-length: 12
+
+PROPS-END
+a
+
+
+Revision-number: 4
+Prop-content-length: 123
+Content-length: 123
+
+K 7
+svn:log
+V 21
+update externals on .
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:09.190834Z
+PROPS-END
+
+Node-path: trunk
+Node-kind: dir
+Node-action: change
+Prop-content-length: 107
+Content-length: 107
+
+K 13
+svn:externals
+V 72
+../externals/project1 deps/project1
+../externals/project2 deps/project2
+
+PROPS-END
+
+
+Revision-number: 5
+Prop-content-length: 115
+Content-length: 115
+
+K 7
+svn:log
+V 13
+add on subdir
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:10.289216Z
+PROPS-END
+
+Node-path: trunk
+Node-kind: dir
+Node-action: change
+Prop-content-length: 71
+Content-length: 71
+
+K 13
+svn:externals
+V 36
+../externals/project2 deps/project2
+
+PROPS-END
+
+
+Node-path: trunk/subdir
+Node-kind: dir
+Node-action: add
+Prop-content-length: 71
+Content-length: 71
+
+K 13
+svn:externals
+V 36
+../externals/project1 deps/project1
+
+PROPS-END
+
+
+Node-path: trunk/subdir2
+Node-kind: dir
+Node-action: add
+Prop-content-length: 71
+Content-length: 71
+
+K 13
+svn:externals
+V 36
+../externals/project1 deps/project1
+
+PROPS-END
+
+
+Revision-number: 6
+Prop-content-length: 125
+Content-length: 125
+
+K 7
+svn:log
+V 23
+remove externals subdir
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:11.184732Z
+PROPS-END
+
+Node-path: trunk/subdir
+Node-action: delete
+
+
+Revision-number: 7
+Prop-content-length: 126
+Content-length: 126
+
+K 7
+svn:log
+V 24
+remove externals subdir2
+K 10
+svn:author
+V 7
+pmezard
+K 8
+svn:date
+V 27
+2008-12-27T12:49:12.194114Z
+PROPS-END
+
+Node-path: trunk/subdir2
+Node-kind: dir
+Node-action: change
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
--- a/tests/run.py
+++ b/tests/run.py
@@ -6,6 +6,7 @@ sys.path.append(os.path.dirname(os.path.
 
 import test_binaryfiles
 import test_diff
+import test_externals
 import test_fetch_branches
 import test_fetch_command
 import test_fetch_command_regexes
@@ -25,6 +26,7 @@ import test_utility_commands
 def suite():
     return unittest.TestSuite([test_binaryfiles.suite(),
                                test_diff.suite(),
+                               test_externals.suite(),
                                test_fetch_branches.suite(),
                                test_fetch_command.suite(),
                                test_fetch_command_regexes.suite(),
new file mode 100644
--- /dev/null
+++ b/tests/test_externals.py
@@ -0,0 +1,87 @@
+import cStringIO
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from mercurial import hg
+from mercurial import ui
+from mercurial import node
+
+import fetch_command
+import svnexternals
+import test_util
+
+
+class TestFetchExternals(test_util.TestBase):
+    def test_externalsfile(self):
+        f = svnexternals.externalsfile()
+        f['t1'] = 'dir1 -r10 svn://foobar'
+        f['t 2'] = 'dir2 -r10 svn://foobar'
+        f['t3'] = ['dir31 -r10 svn://foobar', 'dir32 -r10 svn://foobar']
+        
+        refext = """\
+[t 2]
+ dir2 -r10 svn://foobar
+[t1]
+ dir1 -r10 svn://foobar
+[t3]
+ dir31 -r10 svn://foobar
+ dir32 -r10 svn://foobar
+"""
+        value = f.write()
+        self.assertEqual(refext, value)
+
+        f2 = svnexternals.externalsfile()
+        f2.read(value)
+        self.assertEqual(sorted(f), sorted(f2))
+        for t in f:
+            self.assertEqual(f[t], f2[t])
+
+    def test_externals(self, stupid=False):
+        repo = self._load_fixture_and_fetch('externals.svndump', stupid=stupid)
+
+        ref0 = """\
+[.]
+ ../externals/project1 deps/project1
+"""
+        self.assertEqual(ref0, repo[0]['.hgsvnexternals'].data())
+        ref1 = """\
+[.]
+ ../externals/project1 deps/project1
+ ../externals/project2 deps/project2
+"""
+        self.assertEqual(ref1, repo[1]['.hgsvnexternals'].data())
+
+        ref2 = """\
+[.]
+ ../externals/project2 deps/project2
+[subdir]
+ ../externals/project1 deps/project1
+[subdir2]
+ ../externals/project1 deps/project1
+"""
+        self.assertEqual(ref2, repo[2]['.hgsvnexternals'].data())
+
+        ref3 = """\
+[.]
+ ../externals/project2 deps/project2
+[subdir2]
+ ../externals/project1 deps/project1
+"""
+        self.assertEqual(ref3, repo[3]['.hgsvnexternals'].data())
+
+        ref4 = """\
+[.]
+ ../externals/project2 deps/project2
+"""
+        self.assertEqual(ref4, repo[4]['.hgsvnexternals'].data())
+
+    def test_externals_stupid(self):
+        self.test_externals(True)
+
+def suite():
+    all = [unittest.TestLoader().loadTestsFromTestCase(TestFetchExternals),
+          ]
+    return unittest.TestSuite(all)