Tools/scripts/ftpmirror.py

   1 #! /usr/bin/env python
   2
   3 """Mirror a remote ftp subtree into a local directory tree.
   4
   5 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
   6                  [-l username [-p passwd [-a account]]]
   7                  hostname [remotedir [localdir]]
   8 -v: verbose
   9 -q: quiet
  10 -i: interactive mode
  11 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
  12 -n: don't log in
  13 -r: remove local files/directories no longer pertinent
  14 -l username [-p passwd [-a account]]: login info (default anonymous ftp)
  15 -s pat: skip files matching pattern
  16 hostname: remote host
  17 remotedir: remote directory (default initial)
  18 localdir: local directory (default current)
  19 """
  20
  21 import os
  22 import sys
  23 import time
  24 import getopt
  25 import string
  26 import ftplib
  27 from fnmatch import fnmatch
  28
  29 # Print usage message and exit
  30 def usage(*args):
  31         sys.stdout = sys.stderr
  32         for msg in args: print msg
  33         print __doc__
  34         sys.exit(2)
  35
  36 verbose = 1 # 0 for -q, 2 for -v
  37 interactive = 0
  38 mac = 0
  39 rmok = 0
  40 nologin = 0
  41 skippats = ['.', '..', '.mirrorinfo']
  42
  43 # Main program: parse command line and start processing
  44 def main():
  45         global verbose, interactive, mac, rmok, nologin
  46         try:
  47                 opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
  48         except getopt.error, msg:
  49                 usage(msg)
  50         login = ''
  51         passwd = ''
  52         account = ''
  53         for o, a in opts:
  54                 if o == '-l': login = a
  55                 if o == '-p': passwd = a
  56                 if o == '-a': account = a
  57                 if o == '-v': verbose = verbose + 1
  58                 if o == '-q': verbose = 0
  59                 if o == '-i': interactive = 1
  60                 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
  61                 if o == '-n': nologin = 1
  62                 if o == '-r': rmok = 1
  63                 if o == '-s': skippats.append(a)
  64         if not args: usage('hostname missing')
  65         host = args[0]
  66         remotedir = ''
  67         localdir = ''
  68         if args[1:]:
  69                 remotedir = args[1]
  70                 if args[2:]:
  71                         localdir = args[2]
  72                         if args[3:]: usage('too many arguments')
  73         #
  74         f = ftplib.FTP()
  75         if verbose: print 'Connecting to %s...' % `host`
  76         f.connect(host)
  77         if not nologin:
  78                 if verbose:
  79                         print 'Logging in as %s...' % `login or 'anonymous'`
  80                 f.login(login, passwd, account)
  81         if verbose: print 'OK.'
  82         pwd = f.pwd()
  83         if verbose > 1: print 'PWD =', `pwd`
  84         if remotedir:
  85                 if verbose > 1: print 'cwd(%s)' % `remotedir`
  86                 f.cwd(remotedir)
  87                 if verbose > 1: print 'OK.'
  88                 pwd = f.pwd()
  89                 if verbose > 1: print 'PWD =', `pwd`
  90         #
  91         mirrorsubdir(f, localdir)
  92
  93 # Core logic: mirror one subdirectory (recursively)
  94 def mirrorsubdir(f, localdir):
  95         pwd = f.pwd()
  96         if localdir and not os.path.isdir(localdir):
  97                 if verbose: print 'Creating local directory', `localdir`
  98                 try:
  99                     makedir(localdir)
 100                 except os.error, msg:
 101                     print "Failed to establish local directory", `localdir`
 102                     return
 103         infofilename = os.path.join(localdir, '.mirrorinfo')
 104         try:
 105                 text = open(infofilename, 'r').read()
 106         except IOError, msg:
 107                 text = '{}'
 108         try:
 109                 info = eval(text)
 110         except (SyntaxError, NameError):
 111                 print 'Bad mirror info in %s' % `infofilename`
 112                 info = {}
 113         subdirs = []
 114         listing = []
 115         if verbose: print 'Listing remote directory %s...' % `pwd`
 116         f.retrlines('LIST', listing.append)
 117         filesfound = []
 118         for line in listing:
 119                 if verbose > 1: print '-->', `line`
 120                 if mac:
 121                         # Mac listing has just filenames;
 122                         # trailing / means subdirectory
 123                         filename = string.strip(line)
 124                         mode = '-'
 125                         if filename[-1:] == '/':
 126                                 filename = filename[:-1]
 127                                 mode = 'd'
 128                         infostuff = ''
 129                 else:
 130                         # Parse, assuming a UNIX listing
 131                         words = string.split(line, None, 8)
 132                         if len(words) < 6:
 133                                 if verbose > 1: print 'Skipping short line'
 134                                 continue
 135                         filename = string.lstrip(words[-1])
 136                         i = string.find(filename, " -> ")
 137                         if i >= 0:
 138                                 # words[0] had better start with 'l'...
 139                                 if verbose > 1:
 140                                     print 'Found symbolic link %s' % `filename`
 141                                 linkto = filename[i+4:]
 142                                 filename = filename[:i]
 143                         infostuff = words[-5:-1]
 144                         mode = words[0]
 145                 skip = 0
 146                 for pat in skippats:
 147                         if fnmatch(filename, pat):
 148                                 if verbose > 1:
 149                                         print 'Skip pattern', `pat`,
 150                                         print 'matches', `filename`
 151                                 skip = 1
 152                                 break
 153                 if skip:
 154                         continue
 155                 if mode[0] == 'd':
 156                         if verbose > 1:
 157                                 print 'Remembering subdirectory', `filename`
 158                         subdirs.append(filename)
 159                         continue
 160                 filesfound.append(filename)
 161                 if info.has_key(filename) and info[filename] == infostuff:
 162                         if verbose > 1:
 163                                 print 'Already have this version of',`filename`
 164                         continue
 165                 fullname = os.path.join(localdir, filename)
 166                 tempname = os.path.join(localdir, '@'+filename)
 167                 if interactive:
 168                         doit = askabout('file', filename, pwd)
 169                         if not doit:
 170                                 if not info.has_key(filename):
 171                                         info[filename] = 'Not retrieved'
 172                                 continue
 173                 try:
 174                         os.unlink(tempname)
 175                 except os.error:
 176                         pass
 177                 if mode[0] == 'l':
 178                         if verbose:
 179                                 print "Creating symlink %s -> %s" % (
 180                                         `filename`, `linkto`)
 181                         try:
 182                                 os.symlink(linkto, tempname)
 183                         except IOError, msg:
 184                                 print "Can't create %s: %s" % (
 185                                         `tempname`, str(msg))
 186                                 continue
 187                 else:
 188                         try:
 189                                 fp = open(tempname, 'wb')
 190                         except IOError, msg:
 191                                 print "Can't create %s: %s" % (
 192                                         `tempname`, str(msg))
 193                                 continue
 194                         if verbose:
 195                                 print 'Retrieving %s from %s as %s...' % \
 196                                           (`filename`, `pwd`, `fullname`)
 197                         if verbose:
 198                                 fp1 = LoggingFile(fp, 1024, sys.stdout)
 199                         else:
 200                                 fp1 = fp
 201                         t0 = time.time()
 202                         try:
 203                                 f.retrbinary('RETR ' + filename,
 204                                              fp1.write, 8*1024)
 205                         except ftplib.error_perm, msg:
 206                                 print msg
 207                         t1 = time.time()
 208                         bytes = fp.tell()
 209                         fp.close()
 210                         if fp1 != fp:
 211                                 fp1.close()
 212                 try:
 213                         os.unlink(fullname)
 214                 except os.error:
 215                         pass            # Ignore the error
 216                 try:
 217                         os.rename(tempname, fullname)
 218                 except os.error, msg:
 219                         print "Can't rename %s to %s: %s" % (`tempname`,
 220                                                              `fullname`,
 221                                                              str(msg))
 222                         continue
 223                 info[filename] = infostuff
 224                 writedict(info, infofilename)
 225                 if verbose and mode[0] != 'l':
 226                         dt = t1 - t0
 227                         kbytes = bytes / 1024.0
 228                         print int(round(kbytes)),
 229                         print 'Kbytes in',
 230                         print int(round(dt)),
 231                         print 'seconds',
 232                         if t1 > t0:
 233                                 print '(~%d Kbytes/sec)' % \
 234                                           int(round(kbytes/dt),)
 235                         print
 236         #
 237         # Remove files from info that are no longer remote
 238         deletions = 0
 239         for filename in info.keys():
 240                 if filename not in filesfound:
 241                         if verbose:
 242                                 print "Removing obsolete info entry for",
 243                                 print `filename`, "in", `localdir or "."`
 244                         del info[filename]
 245                         deletions = deletions + 1
 246         if deletions:
 247                 writedict(info, infofilename)
 248         #
 249         # Remove local files that are no longer in the remote directory
 250         try:
 251             if not localdir: names = os.listdir(os.curdir)
 252             else: names = os.listdir(localdir)
 253         except os.error:
 254             names = []
 255         for name in names:
 256                 if name[0] == '.' or info.has_key(name) or name in subdirs:
 257                         continue
 258                 skip = 0
 259                 for pat in skippats:
 260                         if fnmatch(name, pat):
 261                                 if verbose > 1:
 262                                         print 'Skip pattern', `pat`,
 263                                         print 'matches', `name`
 264                                 skip = 1
 265                                 break
 266                 if skip:
 267                         continue
 268                 fullname = os.path.join(localdir, name)
 269                 if not rmok:
 270                         if verbose:
 271                                 print 'Local file', `fullname`,
 272                                 print 'is no longer pertinent'
 273                         continue
 274                 if verbose: print 'Removing local file/dir', `fullname`
 275                 remove(fullname)
 276         #
 277         # Recursively mirror subdirectories
 278         for subdir in subdirs:
 279                 if interactive:
 280                         doit = askabout('subdirectory', subdir, pwd)
 281                         if not doit: continue
 282                 if verbose: print 'Processing subdirectory', `subdir`
 283                 localsubdir = os.path.join(localdir, subdir)
 284                 pwd = f.pwd()
 285                 if verbose > 1:
 286                         print 'Remote directory now:', `pwd`
 287                         print 'Remote cwd', `subdir`
 288                 try:
 289                         f.cwd(subdir)
 290                 except ftplib.error_perm, msg:
 291                         print "Can't chdir to", `subdir`, ":", `msg`
 292                 else:
 293                         if verbose: print 'Mirroring as', `localsubdir`
 294                         mirrorsubdir(f, localsubdir)
 295                         if verbose > 1: print 'Remote cwd ..'
 296                         f.cwd('..')
 297                 newpwd = f.pwd()
 298                 if newpwd != pwd:
 299                         print 'Ended up in wrong directory after cd + cd ..'
 300                         print 'Giving up now.'
 301                         break
 302                 else:
 303                         if verbose > 1: print 'OK.'
 304
 305 # Helper to remove a file or directory tree
 306 def remove(fullname):
 307         if os.path.isdir(fullname) and not os.path.islink(fullname):
 308                 try:
 309                         names = os.listdir(fullname)
 310                 except os.error:
 311                         names = []
 312                 ok = 1
 313                 for name in names:
 314                         if not remove(os.path.join(fullname, name)):
 315                                 ok = 0
 316                 if not ok:
 317                         return 0
 318                 try:
 319                         os.rmdir(fullname)
 320                 except os.error, msg:
 321                         print "Can't remove local directory %s: %s" % \
 322                               (`fullname`, str(msg))
 323                         return 0
 324         else:
 325                 try:
 326                         os.unlink(fullname)
 327                 except os.error, msg:
 328                         print "Can't remove local file %s: %s" % \
 329                               (`fullname`, str(msg))
 330                         return 0
 331         return 1
 332
 333 # Wrapper around a file for writing to write a hash sign every block.
 334 class LoggingFile:
 335         def __init__(self, fp, blocksize, outfp):
 336                 self.fp = fp
 337                 self.bytes = 0
 338                 self.hashes = 0
 339                 self.blocksize = blocksize
 340                 self.outfp = outfp
 341         def write(self, data):
 342                 self.bytes = self.bytes + len(data)
 343                 hashes = int(self.bytes) / self.blocksize
 344                 while hashes > self.hashes:
 345                         self.outfp.write('#')
 346                         self.outfp.flush()
 347                         self.hashes = self.hashes + 1
 348                 self.fp.write(data)
 349         def close(self):
 350                 self.outfp.write('\n')
 351
 352 # Ask permission to download a file.
 353 def askabout(filetype, filename, pwd):
 354         prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
 355         while 1:
 356                 reply = string.lower(string.strip(raw_input(prompt)))
 357                 if reply in ['y', 'ye', 'yes']:
 358                         return 1
 359                 if reply in ['', 'n', 'no', 'nop', 'nope']:
 360                         return 0
 361                 print 'Please answer yes or no.'
 362
 363 # Create a directory if it doesn't exist.  Recursively create the
 364 # parent directory as well if needed.
 365 def makedir(pathname):
 366         if os.path.isdir(pathname):
 367                 return
 368         dirname = os.path.dirname(pathname)
 369         if dirname: makedir(dirname)
 370         os.mkdir(pathname, 0777)
 371
 372 # Write a dictionary to a file in a way that can be read back using
 373 # rval() but is still somewhat readable (i.e. not a single long line).
 374 # Also creates a backup file.
 375 def writedict(dict, filename):
 376         dir, file = os.path.split(filename)
 377         tempname = os.path.join(dir, '@' + file)
 378         backup = os.path.join(dir, file + '~')
 379         try:
 380                 os.unlink(backup)
 381         except os.error:
 382                 pass
 383         fp = open(tempname, 'w')
 384         fp.write('{\n')
 385         for key, value in dict.items():
 386                 fp.write('%s: %s,\n' % (`key`, `value`))
 387         fp.write('}\n')
 388         fp.close()
 389         try:
 390                 os.rename(filename, backup)
 391         except os.error:
 392                 pass
 393         os.rename(tempname, filename)
 394
 395 main()