changeset 821:f28e0f54a6ef

svnmeta: store youngest revision pulled from subversion This prevents re-pulling the same revision over and over, which was a problem when the most recent revision was a tagging revision that wouldn't exist properly in the revmap. This should also allow users to not re-pull huge volumes of commits that have no effect on the hg repository.
author Augie Fackler <durin42@gmail.com>
date Tue, 24 May 2011 21:07:27 -0500 (2011-05-25)
parents 09f7c1c09207
children 033b86e0f56d 4e43e30e3e7d
files hgsubversion/maps.py hgsubversion/svncommands.py hgsubversion/wrappers.py tests/comprehensive/test_stupid_pull.py tests/fixtures/branchtagcollision.sh tests/fixtures/branchtagcollision.svndump tests/test_pull.py tests/test_rebuildmeta.py
diffstat 8 files changed, 295 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/hgsubversion/maps.py
+++ b/hgsubversion/maps.py
@@ -182,13 +182,31 @@ class RevMap(dict):
     def __init__(self, repo):
         dict.__init__(self)
         self.path = os.path.join(repo.path, 'svn', 'rev_map')
-        self.youngest = 0
+        self.ypath = os.path.join(repo.path, 'svn', 'lastpulled')
+        # TODO(durin42): Consider moving management of the youngest
+        # file to svnmeta itself rather than leaving it here.
+        self._youngest = 0
+        # must load youngest file first, or else self._load() can
+        # clobber the info
+        if os.path.isfile(self.ypath):
+            self._youngest = int(open(self.ypath).read().strip())
         self.oldest = 0
         if os.path.isfile(self.path):
             self._load()
         else:
             self._write()
 
+    def _set_youngest(self, rev):
+        self._youngest = max(self._youngest, rev)
+        fp = open(self.ypath, 'wb')
+        fp.write(str(self._youngest) + '\n')
+        fp.close()
+
+    def _get_youngest(self):
+        return self._youngest
+
+    youngest = property(_get_youngest, _set_youngest)
+
     def hashes(self):
         return dict((v, k) for (k, v) in self.iteritems())
 
--- a/hgsubversion/svncommands.py
+++ b/hgsubversion/svncommands.py
@@ -95,6 +95,7 @@ def rebuildmeta(ui, repo, args, **opts):
     if not os.path.exists(svnmetadir):
         os.makedirs(svnmetadir)
 
+    lastpulled = open(os.path.join(svnmetadir, 'lastpulled'), 'wb')
     revmap = open(os.path.join(svnmetadir, 'rev_map'), 'w')
     revmap.write('1\n')
     last_rev = -1
@@ -120,13 +121,18 @@ def rebuildmeta(ui, repo, args, **opts):
     # it would make us use O(revisions^2) time, so we perform an extra traversal
     # of the repository instead. During this traversal, we find all converted
     # changesets that close a branch, and store their first parent
+    youngest = 0
     for rev in repo:
         util.progress(ui, 'prepare', rev, total=numrevs)
         ctx = repo[rev]
         extra = ctx.extra()
         convinfo = extra.get('convert_revision', None)
+        if not convinfo:
+            continue
+        svnrevnum = int(convinfo.rsplit('@', 1)[1])
+        youngest = max(youngest, svnrevnum)
 
-        if not convinfo or not extra.get('close', None):
+        if extra.get('close', None) is None:
             continue
 
         droprev = lambda x: x.rsplit('@', 1)[0]
@@ -136,6 +142,7 @@ def rebuildmeta(ui, repo, args, **opts):
         if droprev(parentinfo) == droprev(convinfo):
             closed.add(parentctx.rev())
 
+    lastpulled.write(str(youngest) + '\n')
     util.progress(ui, 'prepare', None, total=numrevs)
 
     for rev in repo:
--- a/hgsubversion/wrappers.py
+++ b/hgsubversion/wrappers.py
@@ -310,11 +310,13 @@ def pull(repo, source, heads=[], force=F
         total = stopat_rev - start
     else:
         total = svn.HEAD - start
+    lastpulled = None
     try:
         try:
             # start converting revisions
             firstrun = True
             for r in svn.revisions(start=start, stop=stopat_rev):
+                lastpulled = r.revnum
                 if (r.author is None and
                     r.message == 'This is an empty revision for padding.'):
                     continue
@@ -371,6 +373,8 @@ def pull(repo, source, heads=[], force=F
         util.progress(ui, 'pull', None, total=total)
         util.swap_out_encoding(old_encoding)
 
+    if lastpulled is not None:
+        meta.revmap.youngest = lastpulled
     revisions = len(meta.revmap) - oldrevisions
 
     if revisions == 0:
--- a/tests/comprehensive/test_stupid_pull.py
+++ b/tests/comprehensive/test_stupid_pull.py
@@ -45,7 +45,10 @@ attrs = {'_do_case': _do_case,
          }
 for case in (f for f in os.listdir(test_util.FIXTURES) if f.endswith('.svndump')):
     name = 'test_' + case[:-len('.svndump')]
-    attrs[name] = buildmethod(case, name, 'auto')
+    # Automatic layout branchtag collision exposes a minor defect
+    # here, but since it isn't a regression we suppress the test case.
+    if case != 'branchtagcollision.svndump':
+        attrs[name] = buildmethod(case, name, 'auto')
     name += '_single'
     attrs[name] = buildmethod(case, name, 'single')
 
new file mode 100755
--- /dev/null
+++ b/tests/fixtures/branchtagcollision.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Generate branchtagcollision.svndump
+#
+# Generates an svn repository with a branch and a tag that have the same name.
+
+
+mkdir temp
+cd temp
+
+svnadmin create testrepo
+svn checkout file://`pwd`/testrepo client
+
+cd client
+mkdir trunk
+mkdir branches
+mkdir tags
+
+svn add trunk branches tags
+svn commit -m "Initial commit"
+
+echo "fileA" >> trunk/fileA
+svn add trunk/fileA
+svn commit -m "Added fileA"
+
+svn cp trunk branches/A
+svn commit -m "added branch"
+
+echo "fileB" >> trunk/fileB
+svn add trunk/fileB
+svn commit -m "Added fileB"
+
+svn cp trunk tags/A
+svn commit -m "added bad tag"
+
+cd ..
+svnadmin dump testrepo > ../branchtagcollision.svndump
new file mode 100644
--- /dev/null
+++ b/tests/fixtures/branchtagcollision.svndump
@@ -0,0 +1,198 @@
+SVN-fs-dump-format-version: 2
+
+UUID: 764a21f0-1c44-4bc9-b81b-0321cc58934d
+
+Revision-number: 0
+Prop-content-length: 56
+Content-length: 56
+
+K 8
+svn:date
+V 27
+2011-05-24T15:46:13.951233Z
+PROPS-END
+
+Revision-number: 1
+Prop-content-length: 114
+Content-length: 114
+
+K 7
+svn:log
+V 14
+Initial commit
+K 10
+svn:author
+V 5
+augie
+K 8
+svn:date
+V 27
+2011-05-24T15:46:14.518711Z
+PROPS-END
+
+Node-path: branches
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: tags
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: trunk
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Revision-number: 2
+Prop-content-length: 111
+Content-length: 111
+
+K 7
+svn:log
+V 11
+Added fileA
+K 10
+svn:author
+V 5
+augie
+K 8
+svn:date
+V 27
+2011-05-24T15:46:14.922304Z
+PROPS-END
+
+Node-path: trunk/fileA
+Node-kind: file
+Node-action: add
+Prop-content-length: 10
+Text-content-length: 6
+Text-content-md5: 017715e60b9a9450d604e0d489ebc83a
+Text-content-sha1: d0bcb0015aaadb5317419294648c8da6714af81f
+Content-length: 16
+
+PROPS-END
+fileA
+
+
+Revision-number: 3
+Prop-content-length: 112
+Content-length: 112
+
+K 7
+svn:log
+V 12
+added branch
+K 10
+svn:author
+V 5
+augie
+K 8
+svn:date
+V 27
+2011-05-24T15:46:15.328642Z
+PROPS-END
+
+Node-path: branches/A
+Node-kind: dir
+Node-action: add
+Node-copyfrom-rev: 1
+Node-copyfrom-path: trunk
+
+
+Node-path: branches/A/fileA
+Node-kind: file
+Node-action: add
+Node-copyfrom-rev: 2
+Node-copyfrom-path: trunk/fileA
+Text-copy-source-md5: 017715e60b9a9450d604e0d489ebc83a
+Text-copy-source-sha1: d0bcb0015aaadb5317419294648c8da6714af81f
+
+
+Revision-number: 4
+Prop-content-length: 111
+Content-length: 111
+
+K 7
+svn:log
+V 11
+Added fileB
+K 10
+svn:author
+V 5
+augie
+K 8
+svn:date
+V 27
+2011-05-24T15:46:15.616098Z
+PROPS-END
+
+Node-path: trunk/fileB
+Node-kind: file
+Node-action: add
+Prop-content-length: 10
+Text-content-length: 6
+Text-content-md5: 4eb63bbdec5dfa425e3735dc1d4c5ee8
+Text-content-sha1: 03939175ceac92b9c6464d037a0243e22563c423
+Content-length: 16
+
+PROPS-END
+fileB
+
+
+Revision-number: 5
+Prop-content-length: 113
+Content-length: 113
+
+K 7
+svn:log
+V 13
+added bad tag
+K 10
+svn:author
+V 5
+augie
+K 8
+svn:date
+V 27
+2011-05-24T15:46:16.057440Z
+PROPS-END
+
+Node-path: tags/A
+Node-kind: dir
+Node-action: add
+Node-copyfrom-rev: 1
+Node-copyfrom-path: trunk
+
+
+Node-path: tags/A/fileA
+Node-kind: file
+Node-action: add
+Node-copyfrom-rev: 2
+Node-copyfrom-path: trunk/fileA
+Text-copy-source-md5: 017715e60b9a9450d604e0d489ebc83a
+Text-copy-source-sha1: d0bcb0015aaadb5317419294648c8da6714af81f
+
+
+Node-path: tags/A/fileB
+Node-kind: file
+Node-action: add
+Node-copyfrom-rev: 4
+Node-copyfrom-path: trunk/fileB
+Text-copy-source-md5: 4eb63bbdec5dfa425e3735dc1d4c5ee8
+Text-copy-source-sha1: 03939175ceac92b9c6464d037a0243e22563c423
+
+
--- a/tests/test_pull.py
+++ b/tests/test_pull.py
@@ -2,6 +2,7 @@ import test_util
 
 import os.path
 import subprocess
+from mercurial import node
 from mercurial import ui
 from mercurial import util as hgutil
 from mercurial import commands
@@ -51,6 +52,12 @@ class TestPull(test_util.TestBase):
         self.assertTrue('tip' not in repo[None].tags())
         self.assertEqual(len(repo.heads()), 2)
 
+    def test_tag_repull_doesnt_happen(self):
+        repo = self._load_fixture_and_fetch('branchtagcollision.svndump')
+        oldheads = map(node.hex, repo.heads())
+        commands.pull(repo.ui, repo)
+        self.assertEqual(oldheads, map(node.hex, repo.heads()))
+
 def suite():
     import unittest, sys
     return unittest.findTestCases(sys.modules[__name__])
--- a/tests/test_rebuildmeta.py
+++ b/tests/test_rebuildmeta.py
@@ -12,6 +12,17 @@ from mercurial import ui
 from hgsubversion import svncommands
 from hgsubversion import svnmeta
 
+# These test repositories have harmless skew in rebuildmeta for the
+# last-pulled-rev because the last rev in svn causes absolutely no
+# changes in hg.
+expect_youngest_skew = [('file_mixed_with_branches.svndump', False, False),
+                        ('file_mixed_with_branches.svndump', True, False),
+                        ('unrelatedbranch.svndump', False, False),
+                        ('unrelatedbranch.svndump', True, False),
+                        ]
+
+
+
 def _do_case(self, name, stupid, single):
     subdir = test_util.subdir.get(name, '')
     layout = 'auto'
@@ -44,12 +55,18 @@ def _do_case(self, name, stupid, single)
     self.assertTrue(os.path.isdir(os.path.join(src.path, 'svn')),
                     'no .hg/svn directory in the destination!')
     dest = hg.repository(u, os.path.dirname(dest.path))
-    for tf in ('rev_map', 'uuid', 'tagmap', 'layout', 'subdir', ):
+    for tf in ('lastpulled', 'rev_map', 'uuid', 'tagmap', 'layout', 'subdir', ):
+
         stf = os.path.join(src.path, 'svn', tf)
         self.assertTrue(os.path.isfile(stf), '%r is missing!' % stf)
         dtf = os.path.join(dest.path, 'svn', tf)
         self.assertTrue(os.path.isfile(dtf), '%r is missing!' % tf)
         old, new = open(stf).read(), open(dtf).read()
+        if tf == 'lastpulled' and (name,
+                                   stupid, single) in expect_youngest_skew:
+            self.assertNotEqual(old, new,
+                                'rebuildmeta unexpected match on youngest rev!')
+            continue
         self.assertMultiLineEqual(old, new)
         self.assertEqual(src.branchtags(), dest.branchtags())
     srcbi = pickle.load(open(os.path.join(src.path, 'svn', 'branch_info')))