MiniScripts/motherless-dl.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2013 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2013 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2013 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: motherless-dl.py - Last Update: 05/11/2013 Ver. 1.4.0 RC 1 - Author: cooldude2k $
  17 '''
  18
  19 import re, os, sys, httplib, urllib, urllib2, cookielib, StringIO, gzip, time, datetime, argparse, urlparse;
  20
  21 __version_info__ = (1, 3, 0, "RC 3");
  22 if(__version_info__[3]!=None):
  23  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  24 if(__version_info__[3]==None):
  25  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  26
  27 parser = argparse.ArgumentParser();
  28 parser.add_argument("url", help="motherless url");
  29 parser.add_argument("--user-agent", nargs="?", default="Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0", help="specify a custom user agent");
  30 parser.add_argument("--referer", nargs="?", default="http://motherless.com/", help="specify a custom referer, use if the video access");
  31 parser.add_argument("--verbose", action='store_true', help="print various debugging information");
  32 parser.add_argument("--dump-user-agent", action='store_true', help="display the current browser identification");
  33 parser.add_argument("--version", action='store_true', help="print program version and exit");
  34 parser.add_argument("--update", action='store_true', help="update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)");
  35 getargs = parser.parse_args();
  36 if(getargs.version==True):
  37  print(__version__);
  38  sys.exit();
  39 if(getargs.dump_user_agent==True):
  40  print(getargs.user_agent);
  41  sys.exit();
  42 mlessvid = getargs.url;
  43 mlessvid = re.sub(re.escape("http://motherless.com/"), "", mlessvid);
  44 mlessvid = re.sub(re.escape("http://www.motherless.com/"), "", mlessvid);
  45 mlessvid = re.sub(re.escape("motherless.com/"), "", mlessvid);
  46 mlessvid = re.sub(re.escape("www.motherless.com/"), "", mlessvid);
  47 mlessvid = re.sub("^"+re.escape("/"), "", mlessvid);
  48 mlessvid = "http://motherless.com/"+mlessvid;
  49 mregex_text = re.escape("http://motherless.com/")+"([\w\/]+)";
  50 if(re.findall(mregex_text, mlessvid)):
  51  mlessvid = re.findall(mregex_text, mlessvid);
  52  mlessvid = "/"+mlessvid[0];
  53 fakeua = getargs.user_agent;
  54 geturls_cj = cookielib.CookieJar();
  55 geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(geturls_cj));
  56 geturls_opener.addheaders = [("Referer", getargs.referer), ("User-Agent", fakeua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en-CA,en-GB,en-UK,en-AU,en-NZ,en-ZA,en;q=0.5"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")];
  57 per_gal_sleep = 0;
  58 per_url_sleep = 0;
  59 mlessvidid = urlparse.urlparse(mlessvid).path.split('/');
  60 mlessgallist = [];
  61 if((re.match("^random", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^random", mlessvidid[1]) and len(mlessvidid)==3) and (re.match("^image", mlessvidid[2]) or re.match("^video", mlessvidid[2]))):
  62  geturls_text = geturls_opener.open("http://motherless.com"+mlessvid);
  63  mlessvid = geturls_text.geturl();
  64  if(re.findall(mregex_text, mlessvid)):
  65   mlessvid = re.findall(mregex_text, mlessvid);
  66   mlessvid = mlessvid[0];
  67 if((re.match("^galleries", mlessvidid[1]) and len(mlessvidid)==4) or (re.match("^f", mlessvidid[1]) and re.match("^galleries", mlessvidid[2]) and len(mlessvidid)==4)):
  68  geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page=1");
  69  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
  70   strbuf = StringIO.StringIO(geturls_text.read());
  71   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
  72   out_text = gzstrbuf.read()[:];
  73  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
  74   out_text = geturls_text.read()[:];
  75  out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
  76  out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
  77  regex_ptext = re.escape("class=\"pop\" rel=\"")+"([0-9]+)"+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
  78  page_text = re.findall(regex_ptext, out_text);
  79  try:
  80   numpages = int(page_text[-1][0]);
  81  except IndexError:
  82   numpages = 1;
  83  curpage = 1;
  84  while(curpage<=numpages):
  85   if(curpage>1):
  86    geturls_text = geturls_opener.open("http://motherless.com/"+mlessvid+"?page="+str(curpage));
  87    if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
  88     strbuf = StringIO.StringIO(geturls_text.read());
  89     gzstrbuf = gzip.GzipFile(fileobj=strbuf);
  90     out_text = gzstrbuf.read()[:];
  91    if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
  92     out_text = geturls_text.read()[:];
  93    out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
  94    out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
  95   regex_text = re.escape("")+"([\w\/]+)"+re.escape("\" class=\"img-container\" target=\"_self\">");
  96   post_text = re.findall(regex_text, out_text);
  97   numgal = len(post_text);
  98   curgal = 0;
  99   while(curgal<numgal):
 100    mlessgallist.append(post_text[curgal]);
 101    curgal = curgal + 1;
 102   curpage = curpage + 1;
 103 if(not re.match("^galleries", mlessvidid[1]) or (re.match("^galleries", mlessvidid[1]) and len(mlessvidid)<4) or (re.match("^galleries", mlessvidid[1]) and len(mlessvidid)>5)):
 104  mlessgallist.append(mlessvid);
 105 numusrgal = len(mlessgallist);
 106 curusrgal = 0;
 107 while(curusrgal<numusrgal):
 108  mlessvid = mlessgallist[curusrgal];
 109  if(not re.match("^\/", mlessvid)):
 110   mlessvid = "/"+mlessvid;
 111  mlessvidid = urlparse.urlparse(mlessvid).path.split('/');
 112  mlessurllist = [];
 113  if((re.match("^G", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^g", mlessvidid[1]) and len(mlessvidid)==3) or (re.match("^f", mlessvidid[1]) and len(mlessvidid)==4 and (re.match("^videos", mlessvidid[3]) or re.match("^images", mlessvidid[3]))) or (re.match("^live", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^images", mlessvidid[2]) or re.match("^videos", mlessvidid[2]))) or (re.match("^images", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^favorited", mlessvidid[2]) or re.match("^viewed", mlessvidid[2]) or re.match("^commented", mlessvidid[2]) or re.match("^popular", mlessvidid[2]))) or (re.match("^videos", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^favorited", mlessvidid[2]) or re.match("^viewed", mlessvidid[2]) or re.match("^commented", mlessvidid[2]) or re.match("^popular", mlessvidid[2])))):
 114   geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page=1");
 115   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 116    strbuf = StringIO.StringIO(geturls_text.read());
 117    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 118    out_text = gzstrbuf.read()[:];
 119   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 120    out_text = geturls_text.read()[:];
 121   out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 122   out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 123   regex_ptext = re.escape("class=\"pop\" rel=\"")+"([0-9]+)"+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
 124   page_text = re.findall(regex_ptext, out_text);
 125   try:
 126    numpages = int(page_text[-1][0]);
 127   except IndexError:
 128    numpages = 1;
 129   curpage = 1;
 130   while(curpage<=numpages):
 131    if(curpage>1):
 132     geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page="+str(curpage));
 133     if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 134      strbuf = StringIO.StringIO(geturls_text.read());
 135      gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 136      out_text = gzstrbuf.read()[:];
 137     if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 138      out_text = geturls_text.read()[:];
 139    out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 140    out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 141    regex_text = re.escape("")+"([\w\/]+)"+re.escape("\" class=\"img-container\" target=\"_self\">");
 142    post_text = re.findall(regex_text, out_text);
 143    numurls = len(post_text);
 144    cururl = 0;
 145    while(cururl<numurls):
 146     mlessurllist.append(post_text[cururl]);
 147     cururl = cururl + 1;
 148    curpage = curpage + 1;
 149  if((re.match("^G", mlessvidid[1]) and len(mlessvidid)==3 and re.match("([0-9A-F]+)", mlessvidid[2])) or (len(mlessvidid)==2 and re.match("([0-9A-F]+)", mlessvidid[1]))):
 150   mlessurllist.append(mlessvid);
 151  numlist = len(mlessurllist);
 152  curlurl = 0;
 153  while(curlurl<numlist):
 154   geturls_text = geturls_opener.open("http://motherless.com"+mlessurllist[curlurl]);
 155   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 156    strbuf = StringIO.StringIO(geturls_text.read());
 157    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 158    subout_text = gzstrbuf.read()[:];
 159   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 160    subout_text = geturls_text.read()[:];
 161   subout_text = re.sub(re.escape("http://motherless.com"), "", subout_text);
 162   subout_text = re.sub(re.escape("http://www.motherless.com"), "", subout_text);
 163   regex_text = re.escape("__fileurl = '")+"(.*)"+re.escape("';");
 164   post_text = re.findall(regex_text, subout_text);
 165   if(post_text>0):
 166    mlesslink = post_text[0];
 167    mlessext = os.path.splitext(urlparse.urlparse(mlesslink).path)[1];
 168    mlessext = mlessext.replace(".", "");
 169    mlessext = mlessext.lower();
 170    if(mlessext=="mp4" or mlessext=="flv"):
 171     mlesslink = mlesslink+"?start=0";
 172    print(mlesslink);
 173   if(curlurl<(numlist - 1)):
 174    time.sleep(per_url_sleep);
 175   curlurl = curlurl + 1;
 176  if(curusrgal<(numusrgal - 1)):
 177   time.sleep(per_gal_sleep);
 178  curusrgal = curusrgal + 1;
 179
 180  '''
 181  getvidurls_cj = cookielib.CookieJar();
 182  getvidurls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(getvidurls_cj));
 183  getvidurls_opener.addheaders = [("Referer", getargs.referer+mlessvid), ("User-Agent", fakeua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en-CA,en-GB,en-UK,en-AU,en-NZ,en-ZA,en;q=0.5"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")];
 184  getvidurls_text = getvidurls_opener.open(mlesslink);
 185  def chunk_report(bytes_so_far, chunk_size, total_size):
 186   percent = float(bytes_so_far) / total_size;
 187   percent = round(percent*100, 2);
 188   sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" %
 189     (bytes_so_far, total_size, percent));
 190   if bytes_so_far >= total_size:
 191    sys.stdout.write("\n");
 192  def chunk_read(response, chunk_size=8192, report_hook=None):
 193   total_size = response.info().getheader("Content-Length").strip();
 194   total_size = int(total_size);
 195   bytes_so_far = 0;
 196   while 1:
 197    chunk = response.read(chunk_size);
 198    bytes_so_far += len(chunk);
 199    if not chunk:
 200     break;
 201    if report_hook:
 202     report_hook(bytes_so_far, chunk_size, total_size);
 203   return bytes_so_far;
 204  chunk_read(getvidurls_text, report_hook=chunk_report);
 205  vidfile = open(os.getcwd()+os.sep+os.path.basename(urllib2.urlparse.urlsplit(mlesslink)[2]), "wb");
 206  vidfile.write(getvidurls_text.read());
 207  vidfile.close();
 208  '''