Development/gmail.py

   1 #!/usr/bin/python
   2
   3 #Note: We have a custom version of libgmail to fix a bug involving binary attachments (which obviously we have.)
   4
   5 import warnings
   6 warnings.simplefilter("ignore",DeprecationWarning)
   7
   8 import libgmail
   9 import md5
  10 import sys
  11 import passwords
  12 import os
  13 import commands
  14 import re
  15 import time
  16
  17 os.system("rm rawdata_*")
  18
  19 filehashdict = {}
  20
  21 ct = 0
  22 tregex = re.compile("rawdata_([0-9a-f]{32,32})(.*)\.bz2")
  23 outp = commands.getoutput("s3cmd ls s3://questhelper_data/rawdata_")
  24 print "S3 listing snagged"
  25 for line in outp.split('\n'):
  26   if line == "Bucket 's3://questhelper_data':":
  27     continue
  28   serch = tregex.search(line)
  29   if not serch:
  30     print line
  31   toki = serch.group(1)
  32   ext = serch.group(2)
  33   #print toki
  34   filehashdict[toki] = ext
  35 print "Filenames isolated: %d" % len(filehashdict)
  36
  37 ga = libgmail.GmailAccount(passwords.gmail_username, passwords.gmail_password)
  38 ga.login()
  39
  40 destination="./LocalInput/"
  41 label=passwords.gmail_label
  42
  43 argument = "!label:" + label + " has:attachment"
  44 argument = "has:attachment"
  45 inbox=ga.getMessagesByQuery(argument)
  46 i=0
  47
  48 print `len(inbox)`+" messages"
  49 while len(inbox) > 0:
  50   try:
  51     for thread in inbox:
  52       for message in thread:
  53         mark = True
  54         if thread.getLabels().count(label) != 0:
  55           mark = False
  56
  57         clear = True
  58         os.system("date")
  59         print "message "+`i`+" id: "+message.id
  60         #print thread.getLabels()
  61         #print thread.getLabels().count("downloaded")
  62
  63         if True: # we used to make sure it had the right label, or more accurately, didn't
  64           #print 'hoohah'
  65           print '\t'+`len(message.attachments)`+" attachments"
  66           for a in message.attachments:
  67             a.filename = a.filename.encode('ascii', 'ignore').replace('*', '_')
  68             print '\t\t filename:', a.filename
  69             dig=md5.new()
  70             cont=a.content
  71             if cont <> None:
  72               dig.update(cont)
  73               pre=dig.hexdigest()
  74               #dex=filename.find(".")
  75               tup=a.filename.partition(".")
  76               name=pre+tup[1]+tup[2]
  77               f=open(destination+name,"w")
  78               f.write(cont)
  79               f.close()
  80               #message.addLabel("downloaded")
  81
  82               print "\t\t saved"
  83
  84               s3name = "rawdata_" + name + ".bz2"
  85               if not pre in filehashdict:
  86                 # okay, that's cool. Now we S3 it.
  87                 assert(os.system("bzip2 -k --best -c \"%s\" > \"%s\"" % (destination + name, s3name)) == 0)
  88                 assert(os.system("s3cmd put \"%s\" s3://questhelper_data/" % (s3name)) == 0)
  89                 assert(os.system("rm rawdata_*") == 0)
  90                 print "\t\t S3 saved"
  91                 filehashdict[pre] = name.partition(".")[1] + name.partition(".")[2]  # we only look at the first page of emails, over and over. this way, on the second pass through that page, we'll get and delete instead of just re-storing over and over.
  92                 clear = False
  93               else:
  94                 s3oldname = "rawdata_" + pre + filehashdict[pre] + ".bz2"
  95                 if s3oldname != s3name:
  96                   print "\t\t WARNING: Name mismatch! %s vs %s" % (s3name, s3oldname)
  97                 s3cg = "s3cmd --force get \"s3://questhelper_data/%s\" \"%s\"" % (s3oldname, s3oldname)
  98                 while os.system(s3cg) != 0:
  99                   print "\t\t s3cmd failed, sleeping for 15 seconds . . ."
 100                   time.sleep(30)
 101                 assert(os.system("cat \"%s\" | bunzip2 > rawdata_temptest" % (s3oldname)) == 0)
 102                 assert(os.system("diff -q rawdata_temptest \"%s\"" % (destination + name)) == 0)
 103                 assert(os.system("rm rawdata_temptest \"%s\"" % (s3oldname)) == 0)
 104               assert(os.system("rm \"%s\"" % (destination + name)) == 0)
 105             else:
 106               print "foobared attachment"
 107               mark = False
 108               clear = False
 109         if clear:
 110           print "\t Trashing"
 111           ga.trashMessage(message)
 112         i=i+1
 113       if mark:
 114         print "\t Marking"
 115         thread.addLabel(label)
 116   except Exception, e:
 117     raise
 118     #print "whoops"
 119   inbox=ga.getMessagesByQuery(argument)
 120   #print len(inbox)
 121
 122 print `i`+" messages examined and saved"
 123
 124 os.system("rm rawdata_*")