#! /usr/bin/env python """Mirror a remote ftp subtree into a local directory tree. usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]                 [-l username [-p passwd [-a account]]]                 hostname[:port] [remotedir [localdir]]-v: verbose-q: quiet-i: interactive mode-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')-n: don't log in-r: remove local files/directories no longer pertinent-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)-s pat: skip files matching patternhostname: remote host w/ optional port separated by ':'remotedir: remote directory (default initial)localdir: local directory (default current)""" import osimport sysimport timeimport getoptimport ftplibimport netrcfrom fnmatch import fnmatch # Print usage message and exitdef usage(*args):    sys.stdout = sys.stderr    for msg in args: print msg    print __doc__    sys.exit(2) verbose = 1 # 0 for -q, 2 for -vinteractive = 0mac = 0rmok = 0nologin = 0skippats = ['.', '..', '.mirrorinfo'] # Main program: parse command line and start processingdef main():    global verbose, interactive, mac, rmok, nologin    try:        opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')    except getopt.error, msg:        usage(msg)    login = ''    passwd = ''    account = ''    if not args: usage('hostname missing')    host = args[0]    port = 0    if ':' in host:        host, port = host.split(':', 1)        port = int(port)    try:        auth = netrc.netrc().authenticators(host)        if auth is not None:            login, account, passwd = auth    except (netrc.NetrcParseError, IOError):        pass    for o, a in opts:        if o == '-l': login = a        if o == '-p': passwd = a        if o == '-a': account = a        if o == '-v': verbose = verbose + 1        if o == '-q': verbose = 0        if o == '-i': interactive = 1        if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')        if o == '-n': nologin = 1        if o == '-r': rmok = 1        if o == '-s': skippats.append(a)    remotedir = ''    localdir = ''    if args[1:]:        remotedir = args[1]        if args[2:]:            localdir = args[2]            if args[3:]: usage('too many arguments')    #    f = ftplib.FTP()    if verbose: print "Connecting to '%s%s'..." % (host,                                                   (port and ":%d"%port or ""))    f.connect(host,port)    if not nologin:        if verbose:            print 'Logging in as %r...' % (login or 'anonymous')        f.login(login, passwd, account)    if verbose: print 'OK.'    pwd = f.pwd()    if verbose > 1: print 'PWD =', repr(pwd)    if remotedir:        if verbose > 1: print 'cwd(%s)' % repr(remotedir)        f.cwd(remotedir)        if verbose > 1: print 'OK.'        pwd = f.pwd()        if verbose > 1: print 'PWD =', repr(pwd)    #    mirrorsubdir(f, localdir) # Core logic: mirror one subdirectory (recursively)def mirrorsubdir(f, localdir):    pwd = f.pwd()    if localdir and not os.path.isdir(localdir):        if verbose: print 'Creating local directory', repr(localdir)        try:            makedir(localdir)        except os.error, msg:            print "Failed to establish local directory", repr(localdir)            return    infofilename = os.path.join(localdir, '.mirrorinfo')    try:        text = open(infofilename, 'r').read()    except IOError, msg:        text = '{}'    try:        info = eval(text)    except (SyntaxError, NameError):        print 'Bad mirror info in', repr(infofilename)        info = {}    subdirs = []    listing = []    if verbose: print 'Listing remote directory %r...' % (pwd,)    f.retrlines('LIST', listing.append)    filesfound = []    for line in listing:        if verbose > 1: print '-->', repr(line)        if mac:            # Mac listing has just filenames;            # trailing / means subdirectory            filename = line.strip()            mode = '-'            if filename[-1:] == '/':                filename = filename[:-1]                mode = 'd'            infostuff = ''        else:            # Parse, assuming a UNIX listing            words = line.split(None, 8)            if len(words) < 6:                if verbose > 1: print 'Skipping short line'                continue            filename = words[-1].lstrip()            i = filename.find(" -> ")            if i >= 0:                # words[0] had better start with 'l'...                if verbose > 1:                    print 'Found symbolic link %r' % (filename,)                linkto = filename[i+4:]                filename = filename[:i]            infostuff = words[-5:-1]            mode = words[0]        skip = 0        for pat in skippats:            if fnmatch(filename, pat):                if verbose > 1:                    print 'Skip pattern', repr(pat),                    print 'matches', repr(filename)                skip = 1                break        if skip:            continue        if mode[0] == 'd':            if verbose > 1:                print 'Remembering subdirectory', repr(filename)            subdirs.append(filename)            continue        filesfound.append(filename)        if info.has_key(filename) and info[filename] == infostuff:            if verbose > 1:                print 'Already have this version of',repr(filename)            continue        fullname = os.path.join(localdir, filename)        tempname = os.path.join(localdir, '@'+filename)        if interactive:            doit = askabout('file', filename, pwd)            if not doit:                if not info.has_key(filename):                    info[filename] = 'Not retrieved'                continue        try:            os.unlink(tempname)        except os.error:            pass        if mode[0] == 'l':            if verbose:                print "Creating symlink %r -> %r" % (filename, linkto)            try:                os.symlink(linkto, tempname)            except IOError, msg:                print "Can't create %r: %s" % (tempname, msg)                continue        else:            try:                fp = open(tempname, 'wb')            except IOError, msg:                print "Can't create %r: %s" % (tempname, msg)                continue            if verbose:                print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)            if verbose:                fp1 = LoggingFile(fp, 1024, sys.stdout)            else:                fp1 = fp            t0 = time.time()            try:                f.retrbinary('RETR ' + filename,                             fp1.write, 8*1024)            except ftplib.error_perm, msg:                print msg            t1 = time.time()            bytes = fp.tell()            fp.close()            if fp1 != fp:                fp1.close()        try:            os.unlink(fullname)        except os.error:            pass            # Ignore the error        try:            os.rename(tempname, fullname)        except os.error, msg:            print "Can't rename %r to %r: %s" % (tempname, fullname, msg)            continue        info[filename] = infostuff        writedict(info, infofilename)        if verbose and mode[0] != 'l':            dt = t1 - t0            kbytes = bytes / 1024.0            print int(round(kbytes)),            print 'Kbytes in',            print int(round(dt)),            print 'seconds',            if t1 > t0:                print '(~%d Kbytes/sec)' % \                          int(round(kbytes/dt),)            print    #    # Remove files from info that are no longer remote    deletions = 0    for filename in info.keys():        if filename not in filesfound:            if verbose:                print "Removing obsolete info entry for",                print repr(filename), "in", repr(localdir or ".")            del info[filename]            deletions = deletions + 1    if deletions:        writedict(info, infofilename)    #    # Remove local files that are no longer in the remote directory    try:        if not localdir: names = os.listdir(os.curdir)        else: names = os.listdir(localdir)    except os.error:        names = []    for name in names:        if name[0] == '.' or info.has_key(name) or name in subdirs:            continue        skip = 0        for pat in skippats:            if fnmatch(name, pat):                if verbose > 1:                    print 'Skip pattern', repr(pat),                    print 'matches', repr(name)                skip = 1                break        if skip:            continue        fullname = os.path.join(localdir, name)        if not rmok:            if verbose:                print 'Local file', repr(fullname),                print 'is no longer pertinent'            continue        if verbose: print 'Removing local file/dir', repr(fullname)        remove(fullname)    #    # Recursively mirror subdirectories    for subdir in subdirs:        if interactive:            doit = askabout('subdirectory', subdir, pwd)            if not doit: continue        if verbose: print 'Processing subdirectory', repr(subdir)        localsubdir = os.path.join(localdir, subdir)        pwd = f.pwd()        if verbose > 1:            print 'Remote directory now:', repr(pwd)            print 'Remote cwd', repr(subdir)        try:            f.cwd(subdir)        except ftplib.error_perm, msg:            print "Can't chdir to", repr(subdir), ":", repr(msg)        else:            if verbose: print 'Mirroring as', repr(localsubdir)            mirrorsubdir(f, localsubdir)            if verbose > 1: print 'Remote cwd ..'            f.cwd('..')        newpwd = f.pwd()        if newpwd != pwd:            print 'Ended up in wrong directory after cd + cd ..'            print 'Giving up now.'            break        else:            if verbose > 1: print 'OK.' # Helper to remove a file or directory treedef remove(fullname):    if os.path.isdir(fullname) and not os.path.islink(fullname):        try:            names = os.listdir(fullname)        except os.error:            names = []        ok = 1        for name in names:            if not remove(os.path.join(fullname, name)):                ok = 0        if not ok:            return 0        try:            os.rmdir(fullname)        except os.error, msg:            print "Can't remove local directory %r: %s" % (fullname, msg)            return 0    else:        try:            os.unlink(fullname)        except os.error, msg:            print "Can't remove local file %r: %s" % (fullname, msg)            return 0    return 1 # Wrapper around a file for writing to write a hash sign every block.class LoggingFile:    def __init__(self, fp, blocksize, outfp):        self.fp = fp        self.bytes = 0        self.hashes = 0        self.blocksize = blocksize        self.outfp = outfp    def write(self, data):        self.bytes = self.bytes + len(data)        hashes = int(self.bytes) / self.blocksize        while hashes > self.hashes:            self.outfp.write('#')            self.outfp.flush()            self.hashes = self.hashes + 1        self.fp.write(data)    def close(self):        self.outfp.write('\n') # Ask permission to download a file.def askabout(filetype, filename, pwd):    prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)    while 1:        reply = raw_input(prompt).strip().lower()        if reply in ['y', 'ye', 'yes']:            return 1        if reply in ['', 'n', 'no', 'nop', 'nope']:            return 0        print 'Please answer yes or no.' # Create a directory if it doesn't exist.  Recursively create the# parent directory as well if needed.def makedir(pathname):    if os.path.isdir(pathname):        return    dirname = os.path.dirname(pathname)    if dirname: makedir(dirname)    os.mkdir(pathname, 0777) # Write a dictionary to a file in a way that can be read back using# rval() but is still somewhat readable (i.e. not a single long line).# Also creates a backup file.def writedict(dict, filename):    dir, fname = os.path.split(filename)    tempname = os.path.join(dir, '@' + fname)    backup = os.path.join(dir, fname + '~')    try:        os.unlink(backup)    except os.error:        pass    fp = open(tempname, 'w')    fp.write('{\n')    for key, value in dict.items():        fp.write('%r: %r,\n' % (key, value))    fp.write('}\n')    fp.close()    try:        os.rename(filename, backup)    except os.error:        pass    os.rename(tempname, filename)  if __name__ == '__main__':    main()