MiniScripts/motherless-dl.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2013 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2013 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2013 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: motherless-dl.py - Last Update: 10/23/2013 Ver. 1.6.5 RC 3 - Author: cooldude2k $
  17 '''
  18
  19 from __future__ import division, absolute_import, print_function;
  20 import re, os, sys, urllib, urllib2, cookielib, StringIO, gzip, time, datetime, argparse, urlparse;
  21 if(__name__ == "__main__"):
  22  sys.tracebacklimit = 0;
  23 __version_info__ = (1, 6, 5, "RC 3");
  24 __version_date__ = "2013.10.23";
  25 if(__version_info__[3]!=None):
  26  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  27 if(__version_info__[3]==None):
  28  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  29
  30 parser = argparse.ArgumentParser(description="get urls of images/videos from motherless.com", conflict_handler="resolve", add_help=True);
  31 parser.add_argument("url", nargs="*", help="motherless url");
  32 parser.add_argument('-v', '--version', action='version', version=__version__)
  33 parser.add_argument("--update", action='store_true', help="update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)");
  34 parser.add_argument("--dump-user-agent", action='store_true', help="display the current browser identification");
  35 parser.add_argument("--user-agent", nargs="?", default="Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0", help="specify a custom user agent");
  36 parser.add_argument("--referer", nargs="?", default="http://motherless.com/", help="specify a custom referer, use if the video access");
  37 parser.add_argument("--proxy", nargs="?", default=None, help="Use the specified HTTP/HTTPS proxy");
  38 parser.add_argument("--id", action='store_true', help="use only video ID in file name");
  39 parser.add_argument("--get-url", action='store_true', help="simulate, quiet but print URL");
  40 parser.add_argument("--get-pageurl", action='store_true', help="simulate, quiet but print URL");
  41 parser.add_argument("--get-title", action='store_true', help="simulate, quiet but print title");
  42 parser.add_argument("--get-posts", action='store_true', help="simulate, quiet but print user posts");
  43 parser.add_argument("--get-id", action='store_true', help="simulate, quiet but print id");
  44 parser.add_argument("--get-thumbnail", action='store_true', help="simulate, quiet but print thumbnail URL");
  45 parser.add_argument("--get-filename", action='store_true', help="simulate, quiet but print output filename");
  46 parser.add_argument("--get-format", action='store_true', help="simulate, quiet but print file format");
  47 parser.add_argument("--get-type", action='store_true', help="simulate, quiet but print file type");
  48 parser.add_argument("--get-username", action='store_true', help="simulate, quiet but print uploaders username");
  49 parser.add_argument("--get-bbcode", action='store_true', help="simulate, quiet but print bbcode");
  50 parser.add_argument("--get-html", action='store_true', help="simulate, quiet but print html code");
  51 parser.add_argument("--get-dimensions", action='store_true', help="simulate, quiet but print dimensions (width x height)");
  52 parser.add_argument("--get-width", action='store_true', help="simulate, quiet but print width");
  53 parser.add_argument("--get-height", action='store_true', help="simulate, quiet but print height");
  54 parser.add_argument("--get-views", action='store_true', help="simulate, quiet but print number of views");
  55 parser.add_argument("--get-favorites", action='store_true', help="simulate, quiet but print number of favorites");
  56 parser.add_argument("--verbose", action='store_true', help="print various debugging information");
  57 getargs = parser.parse_args();
  58
  59 if(getargs.update==True):
  60  from distutils.version import LooseVersion as VerCheck;
  61  fakeua = getargs.user_agent;
  62  proxycfg = None;
  63  if(getargs.proxy!=None):
  64   proxycfg = urllib2.ProxyHandler({"http": getargs.proxy});
  65  geturls_cj = cookielib.CookieJar();
  66  if(proxycfg==None):
  67   geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(geturls_cj));
  68  if(proxycfg!=None):
  69   geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(geturls_cj), proxycfg);
  70  geturls_opener.addheaders = [("Referer", "https://github.com/GameMaker2k/Python-Scripts/"), ("User-Agent", fakeua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")];
  71  urllib2.install_opener(geturls_opener);
  72  geturls_text = geturls_opener.open("https://raw.github.com/GameMaker2k/Python-Scripts/master/MiniScripts/motherless-dl.py");
  73  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
  74   strbuf = StringIO.StringIO(geturls_text.read());
  75   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
  76   pyfile_text = gzstrbuf.read()[:];
  77  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
  78   pyfile_text = geturls_text.read()[:];
  79  regex_finddate_text = re.escape("__version_date__ = \"")+"([0-9\.]+)"+re.escape("\"");
  80  finddate_text = re.findall(regex_finddate_text, pyfile_text);
  81  regex_findver_text = re.escape("__version_info__ = (")+"([0-9]+)"+re.escape(", ")+"([0-9]+)"+re.escape(", ")+"([0-9]+)"+re.escape(", \"")+"([A-Z0-9 ]+)"+re.escape("\");");
  82  findver_text = re.findall(regex_findver_text, pyfile_text);
  83  ProVerStr = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+__version_info__[3].replace(" ", "").lower();
  84  ProVerCheck = VerCheck(ProVerStr);
  85  ProDateCheck = VerCheck(__version_date__);
  86  NewVerStr = findver_text[0][0]+"."+findver_text[0][1]+"."+findver_text[0][2]+findver_text[0][3].replace(" ", "").lower();
  87  NewVerCheck = VerCheck(NewVerStr);
  88  NewDateCheck = VerCheck(finddate_text[0]);
  89  if(ProVerStr < NewVerCheck and ProDateCheck <= NewDateCheck):
  90   fileopen = open(__file__, "w+");
  91   fileopen.write(pyfile_text);
  92   fileopen.close();
  93  print();
  94  sys.exit();
  95
  96 if(getargs.dump_user_agent==True):
  97  print(getargs.user_agent);
  98  sys.exit();
  99 if(len(getargs.url)==0):
 100  parser.print_help();
 101  sys.exit();
 102 def motherless_dl(mtlessgetargs=vars(getargs)):
 103  fakeua = mtlessgetargs["user_agent"];
 104  proxycfg = None;
 105  if(mtlessgetargs["proxy"]!=None):
 106   proxycfg = urllib2.ProxyHandler({"http": mtlessgetargs["proxy"]});
 107  geturls_cj = cookielib.CookieJar();
 108  if(proxycfg==None):
 109   geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(geturls_cj));
 110  if(proxycfg!=None):
 111   geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(geturls_cj), proxycfg);
 112  geturls_opener.addheaders = [("Referer", mtlessgetargs["referer"]), ("User-Agent", fakeua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")];
 113  urllib2.install_opener(geturls_opener);
 114  per_gal_sleep = 0;
 115  per_url_sleep = 0;
 116  numurlarg = len(mtlessgetargs["url"]);
 117  cururlarg = 0;
 118  while(cururlarg<numurlarg):
 119   mlessvid = mtlessgetargs["url"][cururlarg];
 120   if(re.match("^s([0-9]+)"+re.escape(".motherlessmedia.com"), urlparse.urlparse(mlessvid).hostname)):
 121    geturls_text = geturls_opener.open("http://motherless.com/mogile_api.php?path="+urllib.quote_plus(mlessvid)+"&redirect=1");
 122    mlessvid = geturls_text.geturl();
 123    mregex_text = re.escape("http://motherless.com/")+"([\w\/\?\&\=]+)";
 124    if(re.findall(mregex_text, mlessvid)):
 125     mlessvid = re.findall(mregex_text, mlessvid);
 126     mlessvid = "http://motherless.com/"+mlessvid[0];
 127     if(mtlessgetargs["verbose"]==True):
 128      print(mlessvid);
 129   if(re.match("^"+re.escape("thumbs.motherlessmedia.com"), urlparse.urlparse(mlessvid).hostname)):
 130    mlessvid = re.sub(re.escape("-zoom"), "", mlessvid);
 131    mlessvid = re.sub(re.escape("-strip"), "", mlessvid);
 132    mlessvidtmp = urlparse.urlparse(mlessvid).path.split("/");
 133    mlessvid = "http://motherless.com/"+mlessvidtmp[2];
 134    mregex_text = re.escape("http://motherless.com/")+"([\w\/\?\&\=]+)";
 135    if(re.findall(mregex_text, mlessvid)):
 136     mlessvid = re.findall(mregex_text, mlessvid);
 137     mlessvid = "http://motherless.com/"+mlessvid[0];
 138   mlessvid = re.sub(re.escape("http://motherless.com/"), "", mlessvid);
 139   mlessvid = re.sub(re.escape("http://www.motherless.com/"), "", mlessvid);
 140   mlessvid = re.sub(re.escape("https://motherless.com/"), "", mlessvid);
 141   mlessvid = re.sub(re.escape("https://www.motherless.com/"), "", mlessvid);
 142   mlessvid = re.sub(re.escape("motherless.com/"), "", mlessvid);
 143   mlessvid = re.sub(re.escape("www.motherless.com/"), "", mlessvid);
 144   mlessvid = re.sub("^"+re.escape("/"), "", mlessvid);
 145   mlessvid = "http://motherless.com/"+mlessvid;
 146   mregex_text = re.escape("http://motherless.com/")+"([\w\/\?\&\=]+)";
 147   if(re.findall(mregex_text, mlessvid)):
 148    mlessvid = re.findall(mregex_text, mlessvid);
 149    mlessvid = "/"+mlessvid[0];
 150   mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(mlessvid).query);
 151   mlessvidid = urlparse.urlparse(mlessvid).path.split("/");
 152   mlessgallist = [];
 153   if((re.match("^random", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^random", mlessvidid[1]) and len(mlessvidid)==3) and (re.match("^image", mlessvidid[2]) or re.match("^video", mlessvidid[2]))):
 154    geturls_text = geturls_opener.open("http://motherless.com"+mlessvid);
 155    mlessvid = geturls_text.geturl();
 156    if(re.findall(mregex_text, mlessvid)):
 157     mlessvid = re.findall(mregex_text, mlessvid);
 158     mlessvid = mlessvid[0];
 159     if(mtlessgetargs["verbose"]==True):
 160      print(mlessvid);
 161   if((re.match("^galleries", mlessvidid[1]) and len(mlessvidid)==4) or (re.match("^f", mlessvidid[1]) and re.match("^galleries", mlessvidid[2]) and len(mlessvidid)==4) or (re.match("^term", mlessvidid[1]) and re.match("^galleries", mlessvidid[2]) and len(mlessvidid)==4)):
 162    geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page=1");
 163    if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 164     strbuf = StringIO.StringIO(geturls_text.read());
 165     gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 166     out_text = gzstrbuf.read()[:];
 167    if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 168     out_text = geturls_text.read()[:];
 169    out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 170    out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 171    out_text = re.sub(re.escape("https://motherless.com"), "", out_text);
 172    out_text = re.sub(re.escape("https://www.motherless.com"), "", out_text);
 173    regex_ptext = re.escape("class=\"pop\" rel=\"")+"([0-9]+)"+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
 174    page_text = re.findall(regex_ptext, out_text);
 175    try:
 176     numpages = int(page_text[-1][0]);
 177    except IndexError:
 178     numpages = 1;
 179    curpage = 1;
 180    while(curpage<=numpages):
 181     if(curpage>1):
 182      geturls_text = geturls_opener.open("http://motherless.com/"+mlessvid+"?page="+str(curpage));
 183      if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 184       strbuf = StringIO.StringIO(geturls_text.read());
 185       gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 186       out_text = gzstrbuf.read()[:];
 187      if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 188       out_text = geturls_text.read()[:];
 189      out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 190      out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 191      out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 192      out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 193     regex_text = re.escape("")+"([\w\/]+)"+re.escape("\" class=\"img-container\" target=\"_self\">");
 194     post_text = re.findall(regex_text, out_text);
 195     numgal = len(post_text);
 196     curgal = 0;
 197     while(curgal<numgal):
 198      mlessgallist.append(post_text[curgal]);
 199      if(mtlessgetargs["verbose"]==True):
 200       print(post_text[curgal]);
 201      curgal = curgal + 1;
 202     curpage = curpage + 1;
 203   if(not re.match("^galleries", mlessvidid[1]) or (re.match("^galleries", mlessvidid[1]) and len(mlessvidid)<4) or (re.match("^galleries", mlessvidid[1]) and len(mlessvidid)>5)):
 204    mlessgallist.append(mlessvid);
 205   numusrgal = len(mlessgallist);
 206   curusrgal = 0;
 207   while(curusrgal<numusrgal):
 208    mlessvid = mlessgallist[curusrgal];
 209    if(not re.match("^\/", mlessvid)):
 210     mlessvid = "/"+mlessvid;
 211    mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(mlessvid).query);
 212    mlessvidid = urlparse.urlparse(mlessvid).path.split("/");
 213    mlessurllist = [];
 214    if((re.match("^G", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^H", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^V", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^live", mlessvidid[1]) and len(mlessvidid)==2) or (re.match("^g", mlessvidid[1]) and len(mlessvidid)==3) or (re.match("^u", mlessvidid[1]) and len(mlessvidid)==3) or (re.match("^term", mlessvidid[1]) and (re.match("^videos", mlessvidid[2]) or re.match("^images", mlessvidid[2])) and len(mlessvidid)==4) or (re.match("^f", mlessvidid[1]) and len(mlessvidid)==4 and (re.match("^videos", mlessvidid[3]) or re.match("^images", mlessvidid[3]))) or (re.match("^live", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^images", mlessvidid[2]) or re.match("^videos", mlessvidid[2]))) or (re.match("^images", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^favorited", mlessvidid[2]) or re.match("^viewed", mlessvidid[2]) or re.match("^commented", mlessvidid[2]) or re.match("^popular", mlessvidid[2]))) or (re.match("^videos", mlessvidid[1]) and len(mlessvidid)==3 and (re.match("^favorited", mlessvidid[2]) or re.match("^viewed", mlessvidid[2]) or re.match("^commented", mlessvidid[2]) or re.match("^popular", mlessvidid[2])))):
 215     addtvar = False;
 216     tvaradd = "";
 217     if(re.match("^u", mlessvidid[1]) and len(mlessvidid)==3):
 218      try:
 219       if(mlessvidqstr["t"][0]=="i" or mlessvidqstr["t"][0]=="v"):
 220        tvaradd = "&t="+mlessvidqstr["t"][0];
 221        addtvar = True;
 222      except KeyError:
 223       addtvar = False;
 224      except IndexError:
 225       addtvar = False;
 226     geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page=1"+tvaradd);
 227     if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 228      strbuf = StringIO.StringIO(geturls_text.read());
 229      gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 230      out_text = gzstrbuf.read()[:];
 231     if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 232      out_text = geturls_text.read()[:];
 233     out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 234     out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 235     out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 236     out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 237     regex_ptext = re.escape("class=\"pop\" rel=\"")+"([0-9]+)"+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
 238     page_text = re.findall(regex_ptext, out_text);
 239     try:
 240      numpages = int(page_text[-1][0]);
 241     except IndexError:
 242      numpages = 1;
 243     curpage = 1;
 244     while(curpage<=numpages):
 245      if(curpage>1):
 246       geturls_text = geturls_opener.open("http://motherless.com"+mlessvid+"?page="+str(curpage)+tvaradd);
 247       if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 248        strbuf = StringIO.StringIO(geturls_text.read());
 249        gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 250        out_text = gzstrbuf.read()[:];
 251       if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 252        out_text = geturls_text.read()[:];
 253      out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 254      out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 255      out_text = re.sub(re.escape("http://motherless.com"), "", out_text);
 256      out_text = re.sub(re.escape("http://www.motherless.com"), "", out_text);
 257      if(re.match("^V", mlessvidid[1])):
 258       out_text = re.sub(re.escape("class=\"img-container\" target=\"_self\""), "title=\"motherless link\"", out_text);
 259       out_text = re.sub(re.escape("class=\"pop plain\" target=\"_blank\""), "title=\"motherless link\"", out_text);
 260       regex_text = re.escape("<a href=\"")+"([\w\/]+)"+re.escape("\" title=\"motherless link\">");
 261      if(not re.match("^V", mlessvidid[1])):
 262       regex_text = re.escape("")+"([\w\/]+)"+re.escape("\" class=\"img-container\" target=\"_self\">");
 263      post_text = re.findall(regex_text, out_text);
 264      numurls = len(post_text);
 265      cururl = 0;
 266      while(cururl<numurls):
 267       mlessurllist.append(post_text[cururl]);
 268       if(mtlessgetargs["verbose"]==True):
 269        print(post_text[cururl]);
 270       cururl = cururl + 1;
 271      curpage = curpage + 1;
 272    if((re.match("^G", mlessvidid[1]) and len(mlessvidid)==3 and re.match("([0-9A-F]+)", mlessvidid[2])) or (re.match("^g", mlessvidid[1]) and len(mlessvidid)==4) or (len(mlessvidid)==2 and re.match("([0-9A-F]+)", mlessvidid[1]))):
 273     mlessurllist.append(mlessvid);
 274    numlist = len(mlessurllist);
 275    curlurl = 0;
 276    mlessoutlist = [];
 277    while(curlurl<numlist):
 278     skiplnk = False;
 279     try:
 280      geturls_text = geturls_opener.open("http://motherless.com"+mlessurllist[curlurl]);
 281     except urllib2.HTTPError:
 282      skiplnk = True;
 283     if(skiplnk==False):
 284      if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 285       strbuf = StringIO.StringIO(geturls_text.read());
 286       gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 287       subout_text = gzstrbuf.read()[:];
 288      if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 289       subout_text = geturls_text.read()[:];
 290      subout_text = re.sub(re.escape("http://motherless.com"), "", subout_text);
 291      subout_text = re.sub(re.escape("http://www.motherless.com"), "", subout_text);
 292      subout_text = re.sub(re.escape("http://motherless.com"), "", subout_text);
 293      subout_text = re.sub(re.escape("http://www.motherless.com"), "", subout_text);
 294      regex_title = re.escape("<title>")+"(.*)"+re.escape("</title>");
 295      title_text = re.findall(regex_title, subout_text);
 296      mlesstitle = re.sub(re.escape(" - MOTHERLESS.COM"), "", title_text[0]);
 297      regex_thumb = re.escape("src=&quot;")+"(.*)"+re.escape("&quot;");
 298      thumb_text = re.findall(regex_thumb, subout_text);
 299      mlessthumb = thumb_text[0];
 300      regex_text = re.escape("__fileurl = '")+"(.*)"+re.escape("';");
 301      post_text = re.findall(regex_text, subout_text);
 302      regex_img = re.escape("<meta property=\"og:image\" content=\"")+"(.*)"+re.escape("\">");
 303      img_text = re.findall(regex_img, subout_text);
 304      mlessimg = img_text[0];
 305      regex_mediatype = re.escape("__mediatype = '")+"(.*)"+re.escape("',");
 306      mediatype_text = re.findall(regex_mediatype, subout_text);
 307      regex_altimg = re.escape("<link rel=\"image_src\" type=\"image/")+"(.*)"+re.escape("\" href=\"")+"(.*)"+re.escape("\">");
 308      altimg_text = re.findall(regex_altimg, subout_text);
 309      mlessaltimg = altimg_text[0][1];
 310      regex_usrname = re.escape("<a href=\"/u/")+"([\w]+)"+re.escape("\" class=\"pop plain thumb-member-link-uploads\">Uploads</a>");
 311      usrname_text = re.findall(regex_usrname, subout_text);
 312      mlessusrname = usrname_text[0];
 313      mlessid = re.sub("^"+re.escape("/"), "", mlessurllist[curlurl]);
 314      mlesspurl = "http://motherless.com"+mlessurllist[curlurl];
 315      regex_numviews = re.escape("<strong>Views</strong>")+"\n+\t+([^\t]+)\t+"+re.escape("</h2>");
 316      numviews_text = re.findall(regex_numviews, subout_text);
 317      mlessnumviews = numviews_text[0];
 318      mlessnumviews = re.sub(re.escape(","), "", mlessnumviews);
 319      regex_numfavs = re.escape("<strong>Favorited</strong>")+"\n+\t+([^\t]+)\t+"+re.escape("</h2>");
 320      numfavs_text = re.findall(regex_numfavs, subout_text);
 321      mlessnumfavs = numfavs_text[0];
 322      mlessnumfavs = re.sub(re.escape(","), "", mlessnumfavs);
 323      ''' some good regex "!-%'-?A-~ " "!-%'-?A-~ \<\>\"\'\@\#" '''
 324      regex_postdata = re.escape("<div class=\"media-comment-contents\">")+"\n\t+"+re.escape("<h4>")+"\n\t+"+re.escape("<a href=\"/m/")+"([\w]+)"+re.escape("\" class=\"pop plain\" target=\"_blank\">")+"\n\t+([^\t]+)\t+"+re.escape("</a>")+"\n\t+"+re.escape("</h4>")+"\n\t+"+re.escape("<div class=\"media-comment-meta\">")+"\n\t+([^\t]+)\t+"+re.escape("</div>")+"\n\t+"+re.escape("<div style=\"text-align: justify;\">")+"\n\t+([^\t]+)\t+"+re.escape("</div>");
 325      postdata_text = re.findall(regex_postdata, subout_text);
 326      numpost = len(postdata_text);
 327      regex_servsecs = re.escape("Served by web")+"([0-9]+)"+re.escape(" in ")+"([0-9\.]+)"+re.escape(" seconds");
 328      servsecs_text = re.findall(regex_servsecs, subout_text);
 329      servname = "web"+servsecs_text[0][0];
 330      servsecs = float(servsecs_text[0][1]);
 331      curpost = 0;
 332      mlesspostlist = [];
 333      ''' From Amber @ http://stackoverflow.com/a/9662362 '''
 334      TAG_RE = re.compile(r'<[^>]+>');
 335      while(numpost>0 and curpost<numpost):
 336       newpostext = re.sub(re.escape("<br>"), "\n", postdata_text[curpost][3]);
 337       newpostext = re.sub(re.escape("<br/>"), "\n", newpostext);
 338       newpostext = re.sub(re.escape("<br />"), "\n", newpostext);
 339       newpostext = TAG_RE.sub('', newpostext);
 340       newpostext = re.sub(re.escape("/")+"([\w\/]+)", r"http://motherless.com/\1", newpostext);
 341       mlesspostlist.append({"username": postdata_text[curpost][0], "avatar": "http://avatars.motherlessmedia.com/avatars/member/"+postdata_text[curpost][0]+".jpg", "smallavatar": "http://avatars.motherlessmedia.com/avatars/member/"+postdata_text[curpost][0]+"-small.jpg", "post": newpostext});
 342       curpost = curpost + 1;
 343      if(post_text>0):
 344       mlesslink = post_text[0];
 345       mlessext = os.path.splitext(urlparse.urlparse(mlesslink).path)[1];
 346       mlessext = mlessext.replace(".", "");
 347       mlessext = mlessext.lower();
 348       if(mtlessgetargs["id"]==False):
 349        mlessfname = urlparse.urlsplit(mlesslink).path.split("/")[-1];
 350       if(mtlessgetargs["id"]==True):
 351        mlessfname = re.sub(re.escape("/"), "_", mlessid)+"."+mlessext;
 352       if(not mlessext=="mp4" and not mlessext=="flv"):
 353        imginfo = {};
 354        regex_ii_dimensions = re.escape("style=\"width: ")+"([0-9]+)"+re.escape("px; height: ")+"([0-9]+)"+re.escape("px; border: none;\"");
 355        post_ii_dimensions = re.findall(regex_ii_dimensions, subout_text);
 356        post_ii_width = post_ii_dimensions[0][0];
 357        post_ii_height = post_ii_dimensions[0][1];
 358        imginfo = {"width": int(post_ii_height), "height": int(post_ii_width), "views": int(mlessnumviews), "favorites": int(mlessnumfavs)};
 359       if(mlessext=="mp4" or mlessext=="flv"):
 360        vidinfo = {};
 361        mlesslink = mlesslink+"?start=0";
 362        regex_vi_file = re.escape("\"file\"      : \"")+"(.*)"+re.escape("\",");
 363        post_vi_file = re.findall(regex_vi_file, subout_text);
 364        regex_vi_image = re.escape("\"image\"     : \"")+"(.*)"+re.escape("\",");
 365        post_vi_image = re.findall(regex_vi_image, subout_text);
 366        regex_vi_height = re.escape("\"height\"    : ")+"([0-9]+)"+re.escape(",");
 367        post_vi_height = re.findall(regex_vi_height, subout_text);
 368        regex_vi_width = re.escape("\"width\"     : ")+"([0-9]+)"+re.escape(",");
 369        post_vi_width = re.findall(regex_vi_width, subout_text);
 370        regex_vi_filethumb = re.escape("\"file\": ")+"(.*)"+re.escape(",");
 371        post_vi_filethumb = re.findall(regex_vi_filethumb, subout_text);
 372        regex_vi_kind = re.escape("\"kind\": \"")+"(.*)"+re.escape("\"");
 373        post_vi_kind = re.findall(regex_vi_kind, subout_text);
 374        vidinfo = {"file": post_vi_file[0], "image": post_vi_image[0], "width": int(post_vi_width[0]), "height": int(post_vi_height[0]), "views": int(mlessnumviews), "favorites": int(mlessnumfavs), "filethumb": post_vi_filethumb[0], "thumbstrip": "http://thumbs.motherlessmedia.com/thumbs/"+mlessid+"-strip.jpg", "kind": post_vi_kind[0]};
 375       if(mtlessgetargs["verbose"]==True):
 376        print(mlesslink);
 377       mlesslistitms = {};
 378       mlesslistitms.update({"id": mlessid});
 379       mlesslistitms.update({"title": mlesstitle});
 380       mlesslistitms.update({"format": mlessext});
 381       mlesslistitms.update({"filename": mlessfname});
 382       mlesslistitms.update({"thumbnail": mlessthumb});
 383       mlesslistitms.update({"servername": servname});
 384       mlesslistitms.update({"servingtime": servsecs});
 385       mlesslistitms.update({"mediatype": mediatype_text[0]});
 386       if(not mlessext=="mp4" and not mlessext=="flv"):
 387        mlesslistitms.update({"vidpic": mlesslink});
 388        mlesslistitms.update({"type": "image"});
 389        mlesslistitms.update({"info": imginfo});
 390        mlesslistitms.update({"dimensions": str(imginfo["width"])+"x"+str(imginfo["height"])});
 391        mlesslistitms.update({"width": imginfo["width"]});
 392        mlesslistitms.update({"height": imginfo["height"]});
 393        mlesslistitms.update({"views": imginfo["views"]});
 394        mlesslistitms.update({"favorites": imginfo["favorites"]});
 395       if(mlessext=="mp4" or mlessext=="flv"):
 396        mlesslistitms.update({"vidpic": mlessimg});
 397        mlesslistitms.update({"type": "video"});
 398        mlesslistitms.update({"info": vidinfo});
 399        mlesslistitms.update({"dimensions": str(vidinfo["width"])+"x"+str(vidinfo["height"])});
 400        mlesslistitms.update({"width": vidinfo["width"]});
 401        mlesslistitms.update({"height": vidinfo["height"]});
 402        mlesslistitms.update({"views": vidinfo["views"]});
 403        mlesslistitms.update({"favorites": vidinfo["favorites"]});
 404       mlesslistitms.update({"username": mlessusrname});
 405       mlesslistitms.update({"avatar": "http://avatars.motherlessmedia.com/avatars/member/"+mlessusrname+".jpg"});
 406       mlesslistitms.update({"smallavatar": "http://avatars.motherlessmedia.com/avatars/member/"+mlessusrname+"-small.jpg"});
 407       mlesslistitms.update({"posts": mlesspostlist});
 408       mlesslistitms.update({"pageurl": mlesspurl});
 409       mlesslistitms.update({"url": mlesslink});
 410       mlessoutlist.append(mlesslistitms);
 411     if(curlurl<(numlist - 1)):
 412      time.sleep(per_url_sleep);
 413     curlurl = curlurl + 1;
 414    if(curusrgal<(numusrgal - 1)):
 415     time.sleep(per_gal_sleep);
 416    curusrgal = curusrgal + 1;
 417   cururlarg = cururlarg + 1;
 418  return mlessoutlist;
 419 if(__name__ == "__main__"):
 420  mtlesslinks = motherless_dl();
 421  mtlesslncount = len(mtlesslinks);
 422  mtlesscurln = 0;
 423  while(mtlesscurln<mtlesslncount):
 424   if(getargs.get_id==True):
 425    print(mtlesslinks[mtlesscurln]["id"]);
 426   if(getargs.get_title==True):
 427    print(mtlesslinks[mtlesscurln]["title"]);
 428   if(getargs.get_posts==True):
 429    numpost = len(mtlesslinks[mtlesscurln]["posts"]);
 430    curpost = 0;
 431    mlesspostlist = [];
 432    while(numpost>0 and curpost<numpost):
 433     print(mtlesslinks[mtlesscurln]["posts"][curpost]["username"]+": "+mtlesslinks[mtlesscurln]["posts"][curpost]["post"]);
 434     curpost = curpost + 1;
 435   if(getargs.get_format==True):
 436    print(mtlesslinks[mtlesscurln]["format"]);
 437   if(getargs.get_type==True):
 438    print(mtlesslinks[mtlesscurln]["type"]);
 439   if(getargs.get_filename==True):
 440    print(mtlesslinks[mtlesscurln]["filename"]);
 441   if(getargs.get_thumbnail==True):
 442    print(mtlesslinks[mtlesscurln]["thumbnail"]);
 443    if(mtlesslinks[mtlesscurln]["format"]=="mp4" or mtlesslinks[mtlesscurln]["format"]=="flv"):
 444     print(mtlesslinks[mtlesscurln]["vidpic"]);
 445   if(getargs.get_username==True):
 446    print(mtlesslinks[mtlesscurln]["username"]);
 447   if(getargs.get_pageurl==True):
 448    print(mtlesslinks[mtlesscurln]["pageurl"]);
 449   if(getargs.get_bbcode==True):
 450    print("[URL="+mtlesslinks[mtlesscurln]["pageurl"]+"][IMG]"+mtlesslinks[mtlesscurln]["thumbnail"]+"[/IMG][/URL]");
 451   if(getargs.get_html==True):
 452    print("<a href=\""+mtlesslinks[mtlesscurln]["pageurl"]+"\"><img src=\""+mtlesslinks[mtlesscurln]["thumbnail"]+"\"></a>");
 453   if(getargs.get_dimensions==True):
 454    print(mtlesslinks[mtlesscurln]["dimensions"]);
 455   if(getargs.get_width==True):
 456    print(str(mtlesslinks[mtlesscurln]["width"]));
 457   if(getargs.get_height==True):
 458    print(str(mtlesslinks[mtlesscurln]["height"]));
 459   if(getargs.get_views==True):
 460    print(mtlesslinks[mtlesscurln]["views"]);
 461   if(getargs.get_favorites==True):
 462    print(mtlesslinks[mtlesscurln]["favorites"]);
 463   if(getargs.get_url==True or (getargs.get_id==False and getargs.get_title==False and getargs.get_posts==False and getargs.get_format==False and getargs.get_filename==False and getargs.get_thumbnail==False and getargs.get_username==False and getargs.get_pageurl==False and getargs.get_bbcode==False and getargs.get_html==False and getargs.get_dimensions==False and getargs.get_width==False and getargs.get_height==False and getargs.get_views==False and getargs.get_favorites==False and getargs.get_type==False)):
 464    print(mtlesslinks[mtlesscurln]["url"]);
 465   mtlesscurln = mtlesscurln + 1;