from __future__ import absolute_import, print_function import collectionsimport itertoolsimport ioimport osimport urllibimport re from dulwich.errors import HangupException, GitProtocolErrorfrom dulwich.objects import Blob, Commit, Tag, Tree, parse_timezonefrom dulwich.pack import create_delta, apply_deltafrom dulwich.repo import Repo, check_ref_formatfrom dulwich import clientfrom dulwich import config as dul_configfrom dulwich import diff_tree from mercurial.i18n import _from mercurial.node import hex, bin, nullidfrom mercurial import (    bookmarks,    commands,    context,    encoding,    error,    phases,    util as hgutil,) from . import _sshfrom . import compatfrom . import git2hgfrom . import hg2gitfrom . import utilfrom .overlay import overlayrepo  RE_GIT_AUTHOR = re.compile('^(.*?) ?\<(.*?)(?:\>(.*))?$') RE_GIT_SANITIZE_AUTHOR = re.compile('[<>\n]') RE_GIT_AUTHOR_EXTRA = re.compile('^(.*?)\ ext:\((.*)\) <(.*)\>$') RE_GIT_EXTRA_KEY = re.compile('GIT([0-9]*)-(.*)') # Test for git:// and git+ssh:// URI.# Support several URL forms, including separating the# host and path with either a / or : (sepr)RE_GIT_URI = re.compile(    r'^(?P<scheme>git([+]ssh)?://)(?P<host>.*?)(:(?P<port>\d+))?'    r'(?P<sepr>[:/])(?P<path>.*)$') RE_NEWLINES = re.compile('[\r\n]')RE_GIT_PROGRESS = re.compile('\((\d+)/(\d+)\)') RE_AUTHOR_FILE = re.compile('\s*=\s*') # mercurial.utils.dateutil functions were in mercurial.util in Mercurial < 4.6try:    from mercurial.utils import dateutil    dateutil.parsedateexcept ImportError:    dateutil = hgutil CALLBACK_BUFFER = ''  class GitProgress(object):    """convert git server progress strings into mercurial progress"""    def __init__(self, ui):        self.ui = ui         self._progress = None        self.msgbuf = ''     def progress(self, msg):        # 'Counting objects: 33640, done.\n'        # 'Compressing objects:   0% (1/9955)   \r        msgs = RE_NEWLINES.split(self.msgbuf + msg)        self.msgbuf = msgs.pop()         for msg in msgs:            td = msg.split(':', 1)            data = td.pop()            if not td:                self.flush(data)                continue            topic = td[0]             m = RE_GIT_PROGRESS.search(data)            if m:                if self._progress and self._progress.topic != topic:                    self.flush()                if not self._progress:                    self._progress = compat.makeprogress(self.ui, topic)                 pos, total = map(int, m.group(1, 2))                self._progress.update(pos, total=total)            else:                self.flush(msg)     def flush(self, msg=None):        if self._progress is None:            return        self._progress.complete()        self._progress = None        if msg:            self.ui.note(msg + '\n')  class GitHandler(object):    map_file = 'git-mapfile'    remote_refs_file = 'git-remote-refs'    tags_file = 'git-tags'     def __init__(self, dest_repo, ui):        self.repo = dest_repo        self.ui = ui        self.vfs = self.repo.vfs         # Mercurial >= 3.3:  repo.shared()        if dest_repo.sharedpath != dest_repo.path:            self.vfs = compat.hgvfs(dest_repo.sharedpath)        if compat.config(ui, 'bool', 'git', 'intree'):            self.gitdir = self.repo.wvfs.join('.git')        else:            self.gitdir = self.vfs.join('git')         self.init_author_file()         self.paths = ui.configitems('paths')         self.branch_bookmark_suffix = compat.config(            ui, 'string', 'git', 'branch_bookmark_suffix')         self._map_git_real = None        self._map_hg_real = None        self.load_tags()        self._remote_refs = None     @property    def _map_git(self):        if self._map_git_real is None:            self.load_map()        return self._map_git_real     @property    def _map_hg(self):        if self._map_hg_real is None:            self.load_map()        return self._map_hg_real     @property    def remote_refs(self):        if self._remote_refs is None:            self.load_remote_refs()        return self._remote_refs     @hgutil.propertycache    def git(self):        # Dulwich is going to try and join unicode ref names against        # the repository path to try and read unpacked refs. This        # doesn't match hg's bytes-only view of filesystems, we just        # have to cope with that. As a workaround, try decoding our        # (bytes) path to the repo in hg's active encoding and hope        # for the best.        gitpath = self.gitdir.decode(encoding.encoding, encoding.encodingmode)        # make the git data directory        if os.path.exists(self.gitdir):            return Repo(gitpath)        else:            os.mkdir(self.gitdir)            return Repo.init_bare(gitpath)     def init_author_file(self):        self.author_map = {}        authors_path = compat.config(self.ui, 'string', 'git', 'authors')        if authors_path:            with open(self.repo.wvfs.join(authors_path)) as f:                for line in f:                    line = line.strip()                    if not line or line.startswith('#'):                        continue                    from_, to = RE_AUTHOR_FILE.split(line, 2)                    self.author_map[from_] = to     # FILE LOAD AND SAVE METHODS     def map_set(self, gitsha, hgsha):        self._map_git[gitsha] = hgsha        self._map_hg[hgsha] = gitsha     def map_hg_get(self, gitsha):        return self._map_git.get(gitsha)     def map_git_get(self, hgsha):        return self._map_hg.get(hgsha)     def load_map(self):        map_git_real = {}        map_hg_real = {}        if os.path.exists(self.vfs.join(self.map_file)):            for line in self.vfs(self.map_file):                # format is <40 hex digits> <40 hex digits>\n                if len(line) != 82:                    raise ValueError(                        _('corrupt mapfile: incorrect line length %d') %                        len(line))                gitsha = line[:40]                hgsha = line[41:81]                map_git_real[gitsha] = hgsha                map_hg_real[hgsha] = gitsha        self._map_git_real = map_git_real        self._map_hg_real = map_hg_real     def save_map(self, map_file):        wlock = self.repo.wlock()        try:            file = self.vfs(map_file, 'w+', atomictemp=True)            map_hg = self._map_hg            buf = io.BytesIO()            bwrite = buf.write            for hgsha, gitsha in compat.iteritems(map_hg):                bwrite("%s %s\n" % (gitsha, hgsha))            file.write(buf.getvalue())            buf.close()            # If this complains, atomictempfile no longer has close            file.close()        finally:            wlock.release()     def load_tags(self):        self.tags = {}        if os.path.exists(self.vfs.join(self.tags_file)):            for line in self.vfs(self.tags_file):                sha, name = line.strip().split(' ', 1)                self.tags[name] = sha     def save_tags(self):        file = self.vfs(self.tags_file, 'w+', atomictemp=True)        for name, sha in sorted(compat.iteritems(self.tags)):            if not self.repo.tagtype(name) == 'global':                file.write("%s %s\n" % (sha, name))        # If this complains, atomictempfile no longer has close        file.close()     def load_remote_refs(self):        self._remote_refs = {}        refdir = os.path.join(self.git.path, 'refs', 'remotes')         paths = self.paths        # if paths are set, we should still check 'default'        if not paths:            paths = (('default', None),)         # we avoid using dulwich's refs method because it is incredibly slow;        # on a repo with a few hundred branches and a few thousand tags,        # dulwich took about 200ms        for p in paths:            remotedir = os.path.join(refdir, p[0])            for root, dirs, files in os.walk(remotedir):                for f in files:                    try:                        ref = root.replace(refdir + os.sep, '') + '/'                        node = open(os.path.join(root, f)).read().strip()                        self._remote_refs[ref + f] = bin(self._map_git[node])                    except (KeyError, IOError):                        pass     # END FILE LOAD AND SAVE METHODS     # COMMANDS METHODS     def import_commits(self, remote_name):        refs = self.git.refs.as_dict()        filteredrefs = self.filter_min_date(refs)        self.import_git_objects(remote_name, filteredrefs)        self.update_hg_bookmarks(refs)        self.save_map(self.map_file)     def fetch(self, remote, heads):        result = self.fetch_pack(remote, heads)        remote_name = self.remote_name(remote)         # if remote returns a symref for HEAD, then let's store that        rhead = None        rnode = None        oldheads = self.repo.changelog.heads()        imported = 0        if result.refs:            filteredrefs = self.filter_min_date(self.filter_refs(result.refs,                                                                 heads))            imported = self.import_git_objects(remote_name, filteredrefs)            self.import_tags(result.refs)            self.update_hg_bookmarks(result.refs)             try:                symref = result.symrefs['HEAD']                if symref.startswith('refs/heads'):                    rhead = symref.replace('refs/heads/', '')                 rnode = result.refs['refs/heads/%s' % rhead]                rnode = self._map_git[rnode]                rnode = self.repo[rnode].node()            except KeyError:                # if there is any error make sure to clear the variables                rhead = None                rnode = None             if remote_name:                self.update_remote_branches(remote_name, result.refs)            elif not self.git.refs.as_dict('refs/remotes/'):                # intial cloning                self.update_remote_branches('default', result.refs)                 # "Activate" a tipmost bookmark.                bms = self.repo['tip'].bookmarks()                 # override the 'tipmost' behavior if we know the remote HEAD                if rnode:                    # make sure the bookmark exists; at the point the remote                    # branches has already been set up                    suffix = self.branch_bookmark_suffix or ''                    changes = [(rhead + suffix, rnode)]                    util.updatebookmarks(self.repo, changes)                    bms = [rhead + suffix]                 if bms:                    bookmarks.activate(self.repo, bms[0])         self.save_map(self.map_file)         # also mark public any branches the user specified        blist = [self.repo[branch].node() for branch in                 self.ui.configlist('git', 'public')]        if rnode and self.ui.configbool('hggit', 'usephases'):            blist.append(rnode)         if blist:            lock = self.repo.lock()            try:                tr = self.repo.transaction("phase")                phases.advanceboundary(self.repo, tr, phases.public,                                       blist)                tr.close()            finally:                if tr is not None:                    tr.release()                lock.release()         if imported == 0:            return 0         # code taken from localrepo.py:addchangegroup        dh = 0        if oldheads:            heads = self.repo.changelog.heads()            dh = len(heads) - len(oldheads)            for h in heads:                if h not in oldheads and self.repo[h].closesbranch():                    dh -= 1         if dh < 0:            return dh - 1        else:            return dh + 1     def export_commits(self):        try:            self.export_git_objects()            self.export_hg_tags()            self.update_references()        finally:            self.save_map(self.map_file)     def get_refs(self, remote):        self.export_commits()        client, path = self.get_transport_and_path(remote)        old_refs = {}        new_refs = {}         def changed(refs):            old_refs.update(refs)            exportable = self.get_exportable()            new_refs.update(self.get_changed_refs(refs, exportable, True))            return refs  # always return the same refs to make the send a no-op         try:            client.send_pack(path, changed, lambda have, want: [])             changed_refs = [ref for ref, sha in compat.iteritems(new_refs)                            if sha != old_refs.get(ref)]            new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]            old = {}            for r in old_refs:                old_ref = self.map_hg_get(old_refs[r])                if old_ref:                    old[bin(old_ref)] = 1             return old, new        except (HangupException, GitProtocolError) as e:            raise error.Abort(_("git remote error: ") + str(e))     def push(self, remote, revs, force):        self.export_commits()        old_refs, new_refs = self.upload_pack(remote, revs, force)        remote_name = self.remote_name(remote)         if remote_name and new_refs:            for ref, new_sha in sorted(compat.iteritems(new_refs)):                old_sha = old_refs.get(ref)                if old_sha is None:                    if self.ui.verbose:                        self.ui.note("adding reference %s::%s => GIT:%s\n" %                                     (remote_name, ref, new_sha[0:8]))                    else:                        self.ui.status("adding reference %s\n" % ref)                elif new_sha != old_sha:                    if self.ui.verbose:                        self.ui.note("updating reference %s::%s => GIT:%s\n" %                                     (remote_name, ref, new_sha[0:8]))                    else:                        self.ui.status("updating reference %s\n" % ref)                else:                    self.ui.debug("unchanged reference %s::%s => GIT:%s\n" %                                  (remote_name, ref, new_sha[0:8]))             self.update_remote_branches(remote_name, new_refs)        if old_refs == new_refs:            self.ui.status(_("no changes found\n"))            ret = None        elif len(new_refs) > len(old_refs):            ret = 1 + (len(new_refs) - len(old_refs))        elif len(old_refs) > len(new_refs):            ret = -1 - (len(new_refs) - len(old_refs))        else:            ret = 1        return ret     def clear(self):        mapfile = self.vfs.join(self.map_file)        if os.path.exists(self.gitdir):            for root, dirs, files in os.walk(self.gitdir, topdown=False):                for name in files:                    os.remove(os.path.join(root, name))                for name in dirs:                    os.rmdir(os.path.join(root, name))            os.rmdir(self.gitdir)        if os.path.exists(mapfile):            os.remove(mapfile)     # incoming support    def getremotechanges(self, remote, revs):        self.export_commits()        result = self.fetch_pack(remote.path, revs)         # refs contains all remote refs. Prune to only those requested.        if revs:            reqrefs = {}            for rev in revs:                for n in ('refs/heads/' + rev, 'refs/tags/' + rev):                    if n in result.refs:                        reqrefs[n] = result.refs[n]        else:            reqrefs = result.refs         commits = [bin(c) for c in self.get_git_incoming(reqrefs).commits]         b = overlayrepo(self, commits, result.refs)         return (b, commits, lambda: None)     # CHANGESET CONVERSION METHODS     def export_git_objects(self):        self.ui.note(_("finding hg commits to export\n"))        repo = self.repo        clnode = repo.changelog.node         nodes = (clnode(n) for n in repo)        to_export = (repo[node] for node in nodes if not hex(node) in                     self._map_hg)         todo_total = len(repo) - len(self._map_hg)        topic = 'find commits to export'        unit = 'commits'         with compat.makeprogress(repo.ui, topic, unit, todo_total) as progress:            export = []            for ctx in to_export:                item = hex(ctx.node())                progress.increment(item=item, total=todo_total)                if ctx.extra().get('hg-git', None) != 'octopus':                    export.append(ctx)             total = len(export)            if not total:                return         self.ui.note(_("exporting hg objects to git\n"))

Lines 1–500 of 1731. Load more