Mercurial > hgsubversion
changeset 1470:b6e2bc962536
maps: increase sqlite cache size automatically
Sqlite cache size is 2MB by default. Increasing the cache size can speed up
migration. In a large repo with millions of revisions, changing the cache
size to a big enough value reduces at least 30% of the migration time.
Give the general trends that sacrifices space for speed, this patch auto
tunes the cache size to be big enough to hold the entire database.
| author | Jun Wu <quark@fb.com> |
|---|---|
| date | Tue, 31 May 2016 15:44:11 +0100 |
| parents | 7bb2c6ca4d24 |
| children | 6bc2a2f61923 |
| files | hgsubversion/maps.py |
| diffstat | 1 files changed, 9 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/hgsubversion/maps.py +++ b/hgsubversion/maps.py @@ -708,6 +708,15 @@ class SqliteRevMap(collections.MutableMa self._db = sqlite3.connect(self._dbpath) self._db.text_factory = bytes + # cache size affects random accessing (e.g. index building) + # performance greatly. default is 2MB (2000 KB), we want to have + # a big enough cache that can hold the entire map. + cachesize = 2000 + for path, ratio in [(self._filepath, 1.7), (self._dbpath, 1)]: + if os.path.exists(path): + cachesize += os.stat(path).st_size * ratio // 1000 + self._db.execute('PRAGMA cache_size=%d' % (-cachesize)) + # disable auto-commit. everything is inside a transaction self._db.isolation_level = 'DEFERRED'