tools/Bilibili/Biligrab/biligrab.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Beining --<ACICFG>
   4 # Purpose: Yet another danmaku and video file downloader of Bilibili.
   5 # Created: 11/06/2013
   6 #
   7 # Biligrab is licensed under MIT license (https://github.com/cnbeining/Biligrab/blob/master/LICENSE)
   8 #
   9 # Copyright (c) 2013-2015
  10
  11 '''
  12 Biligrab
  13 Beining@ACICFG
  14 cnbeining[at]gmail.com
  15 http://www.cnbeining.com
  16 https://github.com/cnbeining/Biligrab
  17 MIT license
  18 '''
  19
  20 from ast import literal_eval
  21 import sys
  22 import os
  23 from StringIO import StringIO
  24 import gzip
  25 import urllib
  26 import urllib2
  27 import math
  28 import json
  29 import commands
  30 import subprocess
  31 import hashlib
  32 import getopt
  33 import logging
  34 import traceback
  35 import threading
  36 import Queue
  37 from time import time
  38
  39 from xml.dom.minidom import parseString
  40
  41 try:
  42     from danmaku2ass2 import *
  43 except Exception:
  44     pass
  45
  46 global vid, cid, partname, title, videourl, part_now, is_first_run, APPKEY, SECRETKEY, LOG_LEVEL, VER, LOCATION_DIR, VIDEO_FORMAT, convert_ass, is_export, IS_SLIENT, pages, IS_M3U, FFPROBE_USABLE, QUALITY, IS_FAKE_IP, FAKE_IP
  47
  48 cookies, VIDEO_FORMAT = '', ''
  49 LOG_LEVEL, pages, FFPROBE_USABLE = 0, 0, 0
  50 APPKEY = '6f90a59ac58a4123'
  51 SECRETKEY = 'b78be1fef78c3e7fdc7633e5fd5eee90'
  52 SECRETKEY_MINILOADER = '1c15888dc316e05a15fdd0a02ed6584f'
  53 VER = '0.98.95'
  54 FAKE_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.52 Safari/537.36'
  55 FAKE_HEADER = {
  56     'User-Agent': FAKE_UA,
  57     'Cache-Control': 'no-cache',
  58     'Pragma': 'no-cache',
  59     'pianhao': '%7B%22qing%22%3A%22super%22%2C%22qtudou%22%3A%22real%22%2C%22qyouku%22%3A%22super%22%2C%22q56%22%3A%22super%22%2C%22qcntv%22%3A%22super%22%2C%22qletv%22%3A%22super2%22%2C%22qqiyi%22%3A%22real%22%2C%22qsohu%22%3A%22real%22%2C%22qqq%22%3A%22real%22%2C%22qhunantv%22%3A%22super%22%2C%22qku6%22%3A%22super%22%2C%22qyinyuetai%22%3A%22super%22%2C%22qtangdou%22%3A%22super%22%2C%22qxunlei%22%3A%22super%22%2C%22qsina%22%3A%22high%22%2C%22qpptv%22%3A%22super%22%2C%22qpps%22%3A%22high%22%2C%22qm1905%22%3A%22high%22%2C%22qbokecc%22%3A%22super%22%2C%22q17173%22%3A%22super%22%2C%22qcuctv%22%3A%22super%22%2C%22q163%22%3A%22super%22%2C%22q51cto%22%3A%22high%22%2C%22xia%22%3A%22auto%22%2C%22pop%22%3A%22no%22%2C%22open%22%3A%22no%22%7D'}
  60 LOCATION_DIR = os.path.dirname(os.path.realpath(__file__))
  61
  62 #----------------------------------------------------------------------
  63 def list_del_repeat(list):
  64     """delete repeated items in a list, and keep the order.
  65     http://www.cnblogs.com/infim/archive/2011/03/10/1979615.html"""
  66     l2 = []
  67     [l2.append(i) for i in list if not i in l2]
  68     return(l2)
  69
  70 #----------------------------------------------------------------------
  71 def logging_level_reader(LOG_LEVEL):
  72     """str->int
  73     Logging level."""
  74     return {
  75         'INFO': logging.INFO,
  76         'DEBUG': logging.DEBUG,
  77         'WARNING': logging.WARNING,
  78         'FATAL': logging.FATAL
  79     }.get(LOG_LEVEL)
  80
  81 #----------------------------------------------------------------------
  82 def calc_sign(string):
  83     """str/any->str
  84     return MD5."""
  85     return str(hashlib.md5(str(string).encode('utf-8')).hexdigest())
  86
  87 #----------------------------------------------------------------------
  88 def read_cookie(cookiepath):
  89     """str->list
  90     Original target: set the cookie
  91     Target now: Set the global header"""
  92     global BILIGRAB_HEADER
  93     try:
  94         cookies_file = open(cookiepath, 'r')
  95         cookies = cookies_file.readlines()
  96         cookies_file.close()
  97         # print(cookies)
  98         return cookies
  99     except Exception:
 100         logging.warning('Cannot read cookie, may affect some videos...')
 101         return ['']
 102
 103 #----------------------------------------------------------------------
 104 def clean_name(name):
 105     """str->str
 106     delete all the dramas in the filename."""
 107     return (str(name).strip().replace('\\',' ').replace('/', ' ').replace('&', ' ')).replace('-', ' ')
 108
 109 #----------------------------------------------------------------------
 110 def send_request(url, header, is_fake_ip):
 111     """str,dict,int->str
 112     Send request, and return answer."""
 113     global IS_FAKE_IP
 114     data = ''
 115     if IS_FAKE_IP == 1:
 116         header['X-Forwarded-For'] = FAKE_IP
 117         header['Client-IP'] = FAKE_IP
 118         header['X-Real-IP'] = FAKE_IP
 119     try:
 120         #logging.debug(header)
 121         request = urllib2.Request(url, headers=header)
 122         response = urllib2.urlopen(request)
 123         data = response.read()
 124     except urllib2.HTTPError:
 125         logging.info('ERROR!')
 126         return ''
 127     if response.info().get('Content-Encoding') == 'gzip':
 128         buf = StringIO(response.read())
 129         f = gzip.GzipFile(fileobj=buf)
 130         data = f.read()
 131     #except Exception:
 132         #raise URLOpenException('Cannot open URL! Raw output:\n\n{output}'.format(output = command_result[1]))
 133     #print(request.headers)
 134     logging.debug(data)
 135     return data
 136
 137 #----------------------------------------------------------------------
 138 def mylist_to_aid_list(mylist):
 139     """str/int->list"""
 140     data = send_request('http://www.bilibili.com/mylist/mylist-{mylist}.js'.format(mylist = mylist), FAKE_HEADER, IS_FAKE_IP)
 141     #request = urllib2.Request('http://www.bilibili.com/mylist/mylist-{mylist}.js'.format(mylist = mylist), headers = FAKE_HEADER)
 142     #response = urllib2.urlopen(request)
 143     aid_list = []
 144     #data = response.read()
 145     for i in data.split('\n')[-3].split(','):
 146         if 'aid' in i:
 147             aid_list.append(i.split(':')[1])
 148     return aid_list
 149
 150
 151
 152 #----------------------------------------------------------------------
 153 def find_cid_api(vid, p, cookies):
 154     """find cid and print video detail
 155     str,int?,str->str,str,str,str
 156     TODO: Use json."""
 157     global cid, partname, title, videourl, pages
 158     cid = 0
 159     title , partname , pages, = '', '', ''
 160     if str(p) is '0' or str(p) is '1':
 161         #str2Hash = 'appkey={APPKEY}&id={vid}&type=xml{SECRETKEY}'.format(APPKEY = APPKEY, vid = vid, SECRETKEY = SECRETKEY)
 162         #biliurl = 'https://api.bilibili.com/view?appkey={APPKEY}&id={vid}&type=xml&sign={sign}'.format(APPKEY = APPKEY, vid = vid, SECRETKEY = SECRETKEY, sign = calc_sign(str2Hash))
 163         biliurl = 'https://api.bilibili.com/view?appkey={APPKEY}&id={vid}&type=xml'.format(APPKEY = '8e9fc618fbd41e28', vid = vid, SECRETKEY = SECRETKEY)
 164
 165     else:
 166         #str2Hash = 'appkey={APPKEY}&id={vid}&page={p}&type=xml{SECRETKEY}'.format(APPKEY = APPKEY, vid = vid, p = p, SECRETKEY = SECRETKEY)
 167         #biliurl = 'https://api.bilibili.com/view?appkey={APPKEY}&id={vid}&page={p}&type=xml&sign={sign}'.format(APPKEY = APPKEY, vid = vid, SECRETKEY = SECRETKEY, p = p, sign = calc_sign(str2Hash))
 168         biliurl = 'https://api.bilibili.com/view?appkey={APPKEY}&id={vid}&page={p}&type=xml'.format(APPKEY = '8e9fc618fbd41e28', vid = vid, SECRETKEY = SECRETKEY, p = p)
 169     logging.debug('BiliURL: ' + biliurl)
 170     videourl = 'http://www.bilibili.com/video/av{vid}/index_{p}.html'.format(vid = vid, p = p)
 171     logging.info('Fetching api to read video info...')
 172     data = ''
 173     try:
 174         #request = urllib2.Request(biliurl, headers=BILIGRAB_HEADER)
 175         #response = urllib2.urlopen(request)
 176         #data = response.read()
 177         data = send_request(biliurl, BILIGRAB_HEADER, IS_FAKE_IP)
 178         logging.debug('Bilibili API: ' + data)
 179         dom = parseString(data)
 180         for node in dom.getElementsByTagName('cid'):
 181             if node.parentNode.tagName == "info":
 182                 cid = node.toxml()[5:-6]
 183                 logging.info('cid is ' + cid)
 184                 break
 185         for node in dom.getElementsByTagName('partname'):
 186             if node.parentNode.tagName == "info":
 187                 partname = clean_name(str(node.toxml()[10:-11]))
 188                 logging.info('partname is ' + partname)# no more /\ drama
 189                 break
 190         for node in dom.getElementsByTagName('title'):
 191             if node.parentNode.tagName == "info":
 192                 title = clean_name(str(node.toxml()[7:-8])).decode("utf-8")
 193                 logging.info((u'Title is ' + title).encode(sys.stdout.encoding))
 194         for node in dom.getElementsByTagName('pages'):
 195             if node.parentNode.tagName == "info":
 196                 pages = clean_name(str(node.toxml()[7:-8]))
 197                 logging.info('Total pages is ' + str(pages))
 198         return [cid, partname, title, pages]
 199     except Exception:  # If API failed
 200         logging.warning('Cannot connect to API server! \nIf you think this is wrong, please open an issue at \nhttps://github.com/cnbeining/Biligrab/issues with *ALL* the screen output, \nas well as your IP address and basic system info.\nYou can get these data via "-l".')
 201         logging.debug('API Data: ' + data)
 202         return ['', '', '', '']
 203
 204 #----------------------------------------------------------------------
 205 def find_cid_flvcd(videourl):
 206     """str->None
 207     set cid."""
 208     global vid, cid, partname, title
 209     logging.info('Fetching webpage with raw page...')
 210     #request = urllib2.Request(videourl, headers=FAKE_HEADER)
 211     data = send_request(videourl, FAKE_HEADER, IS_FAKE_IP)
 212     #request.add_header('Accept-encoding', 'gzip')
 213     #try:
 214         #response = urllib2.urlopen(request)
 215     #except urllib2.HTTPError:
 216         #logging.info('ERROR!')
 217         #return ''
 218     #if response.info().get('Content-Encoding') == 'gzip':
 219         #buf = StringIO(response.read())
 220         #f = gzip.GzipFile(fileobj=buf)
 221         #data = f.read()
 222     data_list = data.split('\n')
 223     logging.debug(data)
 224     # Todo: read title
 225     for lines in data_list:
 226         if 'cid=' in lines:
 227             cid = lines.split('&')
 228             cid = cid[0].split('=')
 229             cid = cid[-1]
 230             logging.info('cid is ' + str(cid))
 231             break
 232
 233 #----------------------------------------------------------------------
 234 def check_dependencies(download_software, concat_software, probe_software):
 235     """None->str,str,str
 236     Will give softwares for concat, download and probe.
 237     The detection of Python3 is located at the end of Main function."""
 238     concat_software_list = ['ffmpeg', 'avconv']
 239     download_software_list = ['aria2c', 'axel', 'wget', 'curl']
 240     probe_software_list = ['ffprobe', 'mediainfo']
 241     name_list = [[concat_software,
 242                   concat_software_list],
 243                  [download_software,
 244                   download_software_list],
 245                  [probe_software,
 246                   probe_software_list]]
 247     for name in name_list:
 248         if name[0].strip().lower() not in name[1]:  # Unsupported software
 249             # Set a Unsupported software,  not blank
 250             if len(name[0].strip()) != 0:
 251                 logging.warning('Requested Software not supported!\n         Biligrab only support these following software(s):\n         ' + str(name[1]) + '\n         Trying to find available one...')
 252             for software in name[1]:
 253                 output = commands.getstatusoutput(software + ' --help')
 254                 if str(output[0]) != '32512':  # If exist
 255                     name[0] = software
 256                     break
 257         if name[0] == '':
 258             logging.fatal('Cannot find software in ' + str(name[1]) + ' !')
 259             exit()
 260     return name_list[0][0], name_list[1][0], name_list[2][0]
 261
 262 #----------------------------------------------------------------------
 263 def download_video_link(part_number, download_software, video_link, thread_single_download):
 264     """set->str"""
 265     logging.info('Downloading #{part_number}...'.format(part_number = part_number))
 266     if download_software == 'aria2c':
 267         cmd = 'aria2c -c -U "{FAKE_UA}" -s{thread_single_download} -x{thread_single_download} -k1M --out {part_number}.flv "{video_link}"'
 268     elif download_software == 'wget':
 269         cmd = 'wget -c -A "{FAKE_UA}" -O {part_number}.flv "{video_link}"'
 270     elif download_software == 'curl':
 271         cmd = 'curl -L -C - -A "{FAKE_UA}" -o {part_number}.flv "{video_link}"'
 272     elif download_software == 'axel':
 273         cmd = 'axel -U "{FAKE_UA}" -n {thread_single_download} -o {part_number}.flv "{video_link}"'
 274     cmd = cmd.format(part_number = part_number, video_link = video_link, thread_single_download = thread_single_download, FAKE_UA = FAKE_UA)
 275     logging.debug(cmd)
 276     return cmd
 277
 278 #----------------------------------------------------------------------
 279 def execute_cmd(cmd):
 280     """"""
 281     return_code = subprocess.call(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 282     if return_code != 0:
 283         logging.warning('ERROR')
 284     return return_code
 285
 286 def execute_sysencode_cmd(command):
 287     """execute cmd with sysencoding"""
 288     os.system(command.decode("utf-8").encode(sys.stdout.encoding))
 289
 290 #----------------------------------------------------------------------
 291 def concat_videos(concat_software, vid_num, filename):
 292     """str,str->None"""
 293     global VIDEO_FORMAT,title
 294     if concat_software == 'ffmpeg':
 295         f = open('ff.txt', 'w')
 296         ff = ''
 297         cwd = os.getcwd()
 298         for i in range(vid_num):
 299             ff += 'file \'{cwd}/{i}.flv\'\n'.format(cwd = cwd, i = i)
 300         # ff = ff.encode("utf8")
 301         f.write(ff)
 302         f.close()
 303         logging.debug(ff)
 304         logging.info('Concating videos...')
 305
 306         execute_sysencode_cmd('ffmpeg -f concat -i ff.txt -c copy "' + filename + '".mp4')
 307         VIDEO_FORMAT = 'mp4'
 308         if os.path.isfile((str(i) + '.mp4').decode("utf-8")):
 309             try:
 310                 # os.remove('ff.txt')
 311                 print((str(i) + '.flv').decode("utf-8"))
 312                 os.remove((str(i) + '.flv').decode("utf-8"))
 313                 for i in range(vid_num):
 314                     os.remove((str(i) + '.flv').decode("utf-8"))
 315                     #execute_sysencode_cmd('rm -r ' + str(i) + '.flv')
 316                 logging.info('Done, enjoy yourself!')
 317             except Exception:
 318                 logging.warning('Cannot delete temporary files!')
 319                 return ['']
 320         else:
 321             print('ERROR: Cannot concatenate files, trying to make flv...')
 322             execute_sysencode_cmd('ffmpeg -f concat -i ff.txt -c copy "' + filename + '".flv')
 323             VIDEO_FORMAT = 'flv'
 324             if os.path.isfile((str(i) + '.flv').decode("utf-8")):
 325                 logging.warning('FLV file made. Not possible to mux to MP4, highly likely due to audio format.')
 326                 #execute_sysencode_cmd('rm -r ff.txt')
 327                 # os.remove('ff.txt')
 328                 print(('ff.txt').decode("utf-8"))
 329                 os.remove(('ff.txt').decode("utf-8"))
 330                 for i in range(vid_num):
 331                     #execute_sysencode_cmd('rm -r ' + str(i) + '.flv')
 332                     os.remove((str(i) + '.flv').decode("utf-8"))
 333             else:
 334                 logging.error('Cannot concatenate files!')
 335     elif concat_software == 'avconv':
 336         pass
 337
 338 #----------------------------------------------------------------------
 339 def process_m3u8(url):
 340     """str->list
 341     Only Youku."""
 342     url_list = []
 343     data = send_request(url, FAKE_HEADER, IS_FAKE_IP)
 344     if data == '':
 345         logging.error('Cannot download required m3u8!')
 346         return []
 347     #request = urllib2.Request(url, headers=BILIGRAB_HEADER)
 348     #try:
 349         #response = urllib2.urlopen(request)
 350     #except Exception:
 351         #logging.error('Cannot download required m3u8!')
 352         #return []
 353     #data = response.read()
 354     #logging.debug(data)
 355     data = data.split()
 356     if 'youku' in url:
 357         return [data[4].split('?')[0]]
 358
 359 #----------------------------------------------------------------------
 360 def make_m3u8(video_list):
 361     """list->str
 362     list:
 363     [(VIDEO_URL, TIME_IN_SEC), ...]"""
 364     TARGETDURATION = int(max([i[1] for i in video_list])) + 1
 365     line = '#EXTM3U\n#EXT-X-TARGETDURATION:{TARGETDURATION}\n#EXT-X-VERSION:2\n'.format(TARGETDURATION = TARGETDURATION)
 366     for i in video_list:
 367         line += '#EXTINF:{time}\n{url}\n'.format(time = str(i[1]), url = i[0])
 368     line += '#EXT-X-ENDLIST'
 369     logging.debug('m3u8: ' + line)
 370     return line
 371
 372 #----------------------------------------------------------------------
 373 def find_video_address_html5(vid, p, header):
 374     """str,str,dict->list
 375     Method #3."""
 376     api_url = 'http://www.bilibili.com/m/html5?aid={vid}&page={p}'.format(vid = vid, p = p)
 377     data = send_request(api_url, header, IS_FAKE_IP)
 378     if data == '':
 379         logging.error('Cannot connect to HTML5 API!')
 380         return []
 381     #request = urllib2.Request(api_url, headers=header)
 382     #url_list = []
 383     #try:
 384         #response = urllib2.urlopen(request)
 385     #except Exception:
 386         #logging.error('Cannot connect to HTML5 API!')
 387         #return []
 388     #data = response.read()
 389     #Fix #13
 390     #if response.info().get('Content-Encoding') == 'gzip':
 391         #data = gzip.GzipFile(fileobj=StringIO(data), mode="r").read()
 392     #logging.debug(data)
 393     info = json.loads(data.decode('utf-8'))
 394     raw_url = info['src']
 395     if 'error.mp4' in raw_url:
 396         logging.error('HTML5 API returned ERROR or not available!')
 397         return []  #As in #11
 398     if 'm3u8' in raw_url:
 399         logging.info('Found m3u8, processing...')
 400         return process_m3u8(raw_url)
 401     return [raw_url]
 402
 403 #----------------------------------------------------------------------
 404 def find_video_address_force_original(cid, header):
 405     """str,str->str
 406     Give the original URL, if possible.
 407     Method #2."""
 408         # Force get oriurl
 409     #sign_this = calc_sign('appkey={APPKEY}&cid={cid}{SECRETKEY}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY = SECRETKEY))
 410     api_url = 'http://interface.bilibili.com/player?'
 411     #data = send_request(api_url + 'appkey={APPKEY}&cid={cid}&sign={sign_this}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY = SECRETKEY, sign_this = sign_this), header, IS_FAKE_IP)
 412     data = send_request(api_url + 'appkey={APPKEY}&cid={cid}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY = SECRETKEY), header, IS_FAKE_IP)
 413     #request = urllib2.Request(api_url + 'appkey={APPKEY}&cid={cid}&sign={sign_this}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY = SECRETKEY, sign_this = sign_this), headers=header)
 414     #response = urllib2.urlopen(request)
 415     #data = response.read()
 416     #logging.debug('interface responce: ' + data)
 417     data = data.split('\n')
 418     for l in data:
 419         if 'oriurl' in l:
 420             originalurl = str(l[8:-9])
 421             logging.info('Original URL is ' + originalurl)
 422             return originalurl
 423     logging.warning('Cannot get original URL! Chances are it does not exist.')
 424     return ''
 425
 426 #----------------------------------------------------------------------
 427 def find_link_flvcd(videourl):
 428     """str->list
 429     Used in method 2 and 5."""
 430     logging.info('Finding link via Flvcd...')
 431     data = send_request('http://www.flvcd.com/parse.php?' + urllib.urlencode([('kw', videourl)]) + '&format=super', FAKE_HEADER, IS_FAKE_IP)
 432
 433     #request = urllib2.Request('http://www.flvcd.com/parse.php?' +
 434                               #urllib.urlencode([('kw', videourl)]) + '&format=super', headers=FAKE_HEADER)
 435     #request.add_header('Accept-encoding', 'gzip')
 436     #response = urllib2.urlopen(request)
 437     #data = response.read()
 438     #if response.info().get('Content-Encoding') == 'gzip':
 439         #buf = StringIO(data)
 440         #f = gzip.GzipFile(fileobj=buf)
 441         #data = f.read()
 442     data_list = data.split('\n')
 443     #logging.debug(data)
 444     for items in data_list:
 445         if 'name' in items and 'inf' in items and 'input' in items:
 446             c = items
 447             rawurlflvcd = c[59:-5]
 448             rawurlflvcd = rawurlflvcd.split('|')
 449             return rawurlflvcd
 450
 451 #----------------------------------------------------------------------
 452 def find_video_address_pr(cid, quality, header):
 453     """str,str->list
 454     The API provided by BilibiliPr."""
 455     logging.info('Finding link via BilibiliPr...')
 456     api_url = 'http://pr.lolly.cc/P{quality}?cid={cid}'.format(quality = quality, cid = cid)
 457     data = send_request(api_url, header, IS_FAKE_IP)
 458
 459     #request = urllib2.Request(api_url, headers=header)
 460     #try:
 461         #response = urllib2.urlopen(request, timeout=3)
 462         #data = response.read()
 463     #except Exception:
 464         #logging.warning('No response!')
 465         #return ['ERROR']
 466     #logging.debug('BilibiliPr API: ' + data)
 467     if '!' in data[0:2]:
 468         logging.warning('API returned 404!')
 469         return ['ERROR']
 470     else:
 471         rawurl = []
 472         originalurl = ''
 473         dom = parseString(data)
 474         for node in dom.getElementsByTagName('durl'):
 475             url = node.getElementsByTagName('url')[0]
 476             rawurl.append(url.childNodes[0].data)
 477         return rawurl
 478
 479 #----------------------------------------------------------------------
 480 def find_video_address_normal_api(cid, header, method, convert_m3u = False):
 481     """str,str,str->list
 482     Change in 0.98: Return the file list directly.
 483     Method:
 484     0: Original API
 485     1: CDN API
 486     2: Original URL API - Divided in another function
 487     3: Mobile API - Divided in another function
 488     4: Flvcd - Divided in another function
 489     5: BilibiliPr
 490      [(VIDEO_URL, TIME_IN_SEC), ...]
 491     """
 492     if method == '1':
 493         api_url = 'http://interface.bilibili.com/v_cdn_play?'
 494     else:  #Method 0 or other
 495         api_url = 'http://interface.bilibili.com/playurl?'
 496     if QUALITY == -1:
 497         sign_this = calc_sign('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER))
 498         interface_url = api_url + 'cid={cid}&from=miniplay&player=1&sign={sign_this}'.format(cid = cid, sign_this = sign_this)
 499         #interface_url = api_url + 'appkey={APPKEY}&cid={cid}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY = SECRETKEY)
 500     else:
 501         sign_this = calc_sign('cid={cid}&from=miniplay&player=1&quality={QUALITY}{SECRETKEY_MINILOADER}'.format(APPKEY = APPKEY, cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER, QUALITY = QUALITY))
 502         interface_url = api_url + 'cid={cid}&from=miniplay&player=1&quality={QUALITY}&sign={sign_this}'.format(cid = cid, sign_this = sign_this, QUALITY = QUALITY)
 503
 504     logging.info(interface_url)
 505     data = send_request(interface_url, header, IS_FAKE_IP)
 506     #request = urllib2.Request(interface_url, headers=header)
 507     #logging.debug('Interface: ' + interface_url)
 508     #response = urllib2.urlopen(request)
 509     #data = response.read()
 510     #logging.debug('interface API: ' + data)
 511     for l in data.split('\n'):  # In case shit happens
 512         if 'error.mp4' in l or 'copyright.mp4' in l:
 513             logging.warning('API header may be blocked!')
 514             return ['API_BLOCKED']
 515     rawurl = []
 516     originalurl = ''
 517     dom = parseString(data)
 518     if convert_m3u:
 519         for node in dom.getElementsByTagName('durl'):
 520             length = node.getElementsByTagName('length')[0]
 521             url = node.getElementsByTagName('url')[0]
 522             rawurl.append((url.childNodes[0].data, int(int(length.childNodes[0].data) / 1000) + 1))
 523     else:
 524         for node in dom.getElementsByTagName('durl'):
 525             url = node.getElementsByTagName('url')[0]
 526             rawurl.append(url.childNodes[0].data)
 527     return rawurl
 528
 529 #----------------------------------------------------------------------
 530 def find_link_you_get(videourl):
 531     """str->list
 532     Extract urls with you-get."""
 533     command_result = commands.getstatusoutput('you-get -u {videourl}'.format(videourl = videourl))
 534     logging.debug(command_result)
 535     if command_result[0] != 0:
 536         raise YougetURLException('You-get failed somehow! Raw output:\n\n{output}'.format(output = command_result[1]))
 537     else:
 538         url_list = command_result[1].split('\n')
 539         for k, v in enumerate(url_list):
 540             if v.startswith('http'):
 541                 url_list = url_list[k:]
 542                 break
 543         #url_list = literal_eval(url_list_str)
 544         logging.debug('URL_LIST:{url_list}'.format(url_list = url_list))
 545         return list(url_list)
 546
 547 #----------------------------------------------------------------------
 548 def get_video(oversea, convert_m3u = False):
 549     """str->list
 550     A full parser for getting video.
 551     convert_m3u: [(URL, time_in_sec)]
 552     else: [url,url]"""
 553     rawurl = []
 554     if oversea == '2':
 555         raw_link = find_video_address_force_original(cid, BILIGRAB_HEADER)
 556         rawurl = find_link_flvcd(raw_link)
 557     elif oversea == '3':
 558         rawurl = find_video_address_html5(vid, p, BILIGRAB_HEADER)
 559         if rawurl == []:  #As in #11
 560             rawurl = find_video_address_html5(vid, p, FAKE_HEADER)
 561     elif oversea == '4':
 562         rawurl = find_link_flvcd(videourl)
 563     elif oversea == '5':
 564         rawurl = find_video_address_pr(cid, 1080, BILIGRAB_HEADER)
 565         if '404' in rawurl[0]:
 566             logging.info('Using lower quality...')
 567             rawurl = find_video_address_pr(cid, 720, BILIGRAB_HEADER)
 568             if '404' in rawurl[0]:
 569                 logging.error('Failed!')
 570                 rawurl = []
 571             else:
 572                 pass
 573         elif 'ERROR' in rawurl[0]:
 574             logging.info('Wait a little bit...')
 575             time.sleep(5)
 576             rawurl = find_video_address_pr(cid, 1080, BILIGRAB_HEADER)
 577     elif oversea == '6':
 578         raw_link = find_video_address_force_original(cid, BILIGRAB_HEADER)
 579         rawurl = find_link_you_get(raw_link)
 580     else:
 581         rawurl = find_video_address_normal_api(cid, BILIGRAB_HEADER, oversea, convert_m3u)
 582         if 'API_BLOCKED' in rawurl[0]:
 583             logging.warning('API header may be blocked! Using fake one instead...')
 584             rawurl = find_video_address_normal_api(cid, FAKE_HEADER, oversea, convert_m3u)
 585     return rawurl
 586
 587 #----------------------------------------------------------------------
 588 def get_resolution(filename, probe_software):
 589     """str,str->list"""
 590     resolution = []
 591     filename = filename + '.' + VIDEO_FORMAT
 592     try:
 593         if probe_software == 'mediainfo':
 594             resolution = get_resolution_mediainfo(filename)
 595         if probe_software == 'ffprobe':
 596             resolution = get_resolution_ffprobe(filename)
 597         logging.debug('Software: {probe_software}, resolution {resolution}'.format(probe_software = probe_software, resolution = resolution))
 598         return resolution
 599     except Exception:  # magic number
 600         return[1280, 720]
 601
 602 #----------------------------------------------------------------------
 603 def get_resolution_mediainfo(filename):
 604     """str->list
 605     [640,360]
 606     path to dimention"""
 607     resolution = str(os.popen('mediainfo \'--Inform=Video;%Width%x%Height%\' "' +filename +'"').read()).strip().split('x')
 608     return [int(resolution[0]), int(resolution[1])]
 609
 610 #----------------------------------------------------------------------
 611 def get_resolution_ffprobe(filename):
 612     '''str->list
 613     [640,360]'''
 614     width = ''
 615     height = ''
 616     cmnd = ['ffprobe', '-show_format', '-show_streams', '-pretty', '-loglevel', 'quiet', filename]
 617     p = subprocess.Popen(cmnd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 618     # print filename
 619     out, err = p.communicate()
 620     if err:
 621         print err
 622         return None
 623     try:
 624         for line in out.split():
 625             if 'width=' in line:
 626                 width = line.split('=')[1]
 627             if 'height=' in line:
 628                 height = line.split('=')[1]
 629     except Exception:
 630         return None
 631     # return width + 'x' + height
 632     return [int(width), int(height)]
 633
 634 #----------------------------------------------------------------------
 635 def get_url_size(url):
 636     """str->int
 637     Get remote URL size by reading Content-Length.
 638     In bytes."""
 639     site = urllib.urlopen(url)
 640     meta = site.info()
 641     return int(meta.getheaders("Content-Length")[0])
 642
 643 #----------------------------------------------------------------------
 644 def getvideosize(url, verbose=False):
 645     try:
 646         if url.startswith('http:') or url.startswith('https:'):
 647             ffprobe_command = ['ffprobe', '-icy', '0', '-loglevel', 'repeat+warning' if verbose else 'repeat+error', '-print_format', 'json', '-select_streams', 'v', '-show_format', '-show_streams', '-timeout', '60000000', '-user-agent', BILIGRAB_UA, url]
 648         else:
 649             ffprobe_command = ['ffprobe', '-loglevel', 'repeat+warning' if verbose else 'repeat+error', '-print_format', 'json', '-select_streams', 'v', '-show_streams', url]
 650         logcommand(ffprobe_command)
 651         ffprobe_process = subprocess.Popen(ffprobe_command, stdout=subprocess.PIPE)
 652         try:
 653             ffprobe_output = json.loads(ffprobe_process.communicate()[0].decode('utf-8', 'replace'))
 654         except KeyboardInterrupt:
 655             logging.warning('Cancelling getting video size, press Ctrl-C again to terminate.')
 656             ffprobe_process.terminate()
 657             return 0, 0
 658         width, height, widthxheight, duration, total_bitrate = 0, 0, 0, 0, 0
 659         try:
 660             if dict.get(ffprobe_output, 'format')['duration'] > duration:
 661                 duration = dict.get(ffprobe_output, 'format')['duration']
 662         except Exception:
 663             pass
 664         for stream in dict.get(ffprobe_output, 'streams', []):
 665             try:
 666                 if duration == 0 and (dict.get(stream, 'duration') > duration):
 667                         duration = dict.get(stream, 'duration')
 668                 if dict.get(stream, 'width')*dict.get(stream, 'height') > widthxheight:
 669                     width, height = dict.get(stream, 'width'), dict.get(stream, 'height')
 670                 if dict.get(stream, 'bit_rate') > total_bitrate:
 671                     total_bitrate += int(dict.get(stream, 'bit_rate'))
 672             except Exception:
 673                 pass
 674         if duration == 0:
 675             duration = int(get_url_size(url) * 8 / total_bitrate)
 676         return [[int(width), int(height)], int(float(duration))+1]
 677     except Exception as e:
 678         logorraise(e)
 679         return [[0, 0], 0]
 680
 681 #----------------------------------------------------------------------
 682 def convert_ass_py3(filename, probe_software, resolution = [0, 0]):
 683     """str,str->None
 684     With danmaku2ass, branch master.
 685     https://github.com/m13253/danmaku2ass/
 686     Author: @m13253
 687     GPLv3
 688     A simple way to do that.
 689     resolution_str:1920x1080"""
 690     xml_name = os.path.abspath(filename + '.xml')
 691     ass_name = filename + '.ass'
 692     logging.info('Converting danmaku to ASS file with danmaku2ass(main)...')
 693     logging.info('Resolution is %dx%d' % (resolution[0], resolution[1]))
 694     if resolution == [0, 0]:
 695         logging.info('Trying to get resolution...')
 696         resolution = get_resolution(filename, probe_software)
 697     logging.info('Resolution is %dx%d' % (resolution[0], resolution[1]))
 698     if execute_sysencode_cmd('python3 %s/danmaku2ass3.py -o %s -s %dx%d -fs %d -a 0.8 -dm 8 %s' % (LOCATION_DIR, ass_name, resolution[0], resolution[1], int(math.ceil(resolution[1] / 21.6)), xml_name)) == 0:
 699         logging.info('The ASS file should be ready!')
 700     else:
 701         logging.error('''Danmaku2ASS failed.
 702         Head to https://github.com/m13253/danmaku2ass/issues to complain about this.''')
 703
 704 #----------------------------------------------------------------------
 705 def convert_ass_py2(filename, probe_software, resolution = [0, 0]):
 706     """str,str->None
 707     With danmaku2ass, branch py2.
 708     https://github.com/m13253/danmaku2ass/tree/py2
 709     Author: @m13253
 710     GPLv3"""
 711     logging.info('Converting danmaku to ASS file with danmaku2ass(py2)...')
 712     xml_name = filename + '.xml'
 713     if resolution == [0, 0]:
 714         logging.info('Trying to get resolution...')
 715         resolution = get_resolution(filename, probe_software)
 716     logging.info('Resolution is {width}x{height}'.format(width = resolution[0], height = resolution[1]))
 717     #convert_ass(xml_name, filename + '.ass', resolution)
 718     try:
 719         Danmaku2ASS(xml_name, filename + '.ass', resolution[0], resolution[1],
 720                     font_size = int(math.ceil(resolution[1] / 21.6)), text_opacity=0.8, duration_marquee=8.0)
 721         logging.info('INFO: The ASS file should be ready!')
 722     except Exception as e:
 723         logging.error('''Danmaku2ASS failed: %s
 724         Head to https://github.com/m13253/danmaku2ass/issues to complain about this.'''% e)
 725         logging.debug(traceback.print_exc())
 726         pass  #Or it may stop leaving lots of lines unprocessed
 727
 728 #----------------------------------------------------------------------
 729 def download_danmaku(cid, filename):
 730     """str,str,int->None
 731     Download XML file, and convert to ASS(if required)
 732     Used to be in main(), but replaced due to the merge of -m (BiligrabLite).
 733     If danmaku only, will see whether need to export ASS."""
 734     logging.info('Fetching XML...')
 735     execute_sysencode_cmd('curl -o "{filename}.xml" --compressed  http://comment.bilibili.com/{cid}.xml'.format(filename = filename, cid = cid))
 736     #execute_sysencode_cmd('gzip -d '+cid+'.xml.gz')
 737     logging.info('The XML file, {filename}.xml should be ready...enjoy!'.format(filename = filename.decode("utf-8").encode(sys.stdout.encoding)))
 738
 739 #----------------------------------------------------------------------
 740 def logcommand(command_line):
 741     logging.debug('Executing: '+' '.join('\''+i+'\'' if ' ' in i or '&' in i or '"' in i else i for i in command_line))
 742
 743 #----------------------------------------------------------------------
 744 def logorraise(message, debug=False):
 745     if debug:
 746         raise message
 747     else:
 748         logging.error(str(message))
 749
 750 ########################################################################
 751 class DanmakuOnlyException(Exception):
 752
 753     '''Deal with DanmakuOnly to stop the main() function.'''
 754     #----------------------------------------------------------------------
 755
 756     def __init__(self, value):
 757         self.value = value
 758     #----------------------------------------------------------------------
 759
 760     def __str__(self):
 761         return repr(self.value)
 762
 763 ########################################################################
 764 class Danmaku2Ass2Exception(Exception):
 765
 766     '''Deal with Danmaku2ASS2 to stop the main() function.'''
 767     #----------------------------------------------------------------------
 768
 769     def __init__(self, value):
 770         self.value = value
 771     #----------------------------------------------------------------------
 772
 773     def __str__(self):
 774         return repr(self.value)
 775
 776 ########################################################################
 777 class NoCidException(Exception):
 778
 779     '''Deal with no cid to stop the main() function.'''
 780     #----------------------------------------------------------------------
 781
 782     def __init__(self, value):
 783         self.value = value
 784     #----------------------------------------------------------------------
 785
 786     def __str__(self):
 787         return repr(self.value)
 788
 789 ########################################################################
 790 class NoVideoURLException(Exception):
 791
 792     '''Deal with no video URL to stop the main() function.'''
 793     #----------------------------------------------------------------------
 794
 795     def __init__(self, value):
 796         self.value = value
 797     #----------------------------------------------------------------------
 798
 799     def __str__(self):
 800         return repr(self.value)
 801
 802 ########################################################################
 803 class ExportM3UException(Exception):
 804
 805     '''Deal with export to m3u to stop the main() function.'''
 806     #----------------------------------------------------------------------
 807
 808     def __init__(self, value):
 809         self.value = value
 810     #----------------------------------------------------------------------
 811
 812     def __str__(self):
 813         return repr(self.value)
 814
 815 ########################################################################
 816 class YougetURLException(Exception):
 817
 818     '''you-get cannot get URL somehow'''
 819     #----------------------------------------------------------------------
 820
 821     def __init__(self, value):
 822         self.value = value
 823     #----------------------------------------------------------------------
 824
 825     def __str__(self):
 826         return repr(self.value)
 827
 828 ########################################################################
 829 class URLOpenException(Exception):
 830
 831     '''cannot get URL somehow'''
 832     #----------------------------------------------------------------------
 833
 834     def __init__(self, value):
 835         self.value = value
 836     #----------------------------------------------------------------------
 837
 838     def __str__(self):
 839         return repr(self.value)
 840
 841
 842 ########################################################################
 843 class DownloadVideo(threading.Thread):
 844     """Threaded Download Video"""
 845     #----------------------------------------------------------------------
 846     def __init__(self, queue):
 847         threading.Thread.__init__(self)
 848         self.queue = queue
 849     #----------------------------------------------------------------------
 850     def run(self):
 851         while True:
 852             #grabs start time from queue
 853             down_set = self.queue.get()
 854             #return_value = download_video(down_set)
 855             cmd = download_video_link(*down_set)
 856             return_value = execute_cmd(cmd)
 857             self.queue.task_done()
 858
 859 #----------------------------------------------------------------------
 860 def main_threading(download_thread = 3, video_list = [], thread_single_download = 16):
 861     """"""
 862     command_pool = [(video_list.index(url_this), download_software, url_this, thread_single_download) for url_this in video_list]
 863     #spawn a pool of threads, and pass them queue instance
 864     for i in range(int(download_thread)):
 865         t = DownloadVideo(queue)
 866         t.setDaemon(True)
 867         t.start()
 868     #populate queue with data
 869     for command_single in command_pool:
 870         queue.put(command_single)
 871     #wait on the queue until everything has been processed
 872     queue.join()
 873
 874 #----------------------------------------------------------------------
 875 def main(vid, p, oversea, cookies, download_software, concat_software, is_export, probe_software, danmaku_only, time_fetch=5, download_thread= 16, thread_single_download= 16):
 876     global cid, partname, title, videourl, is_first_run
 877     videourl = 'http://www.bilibili.com/video/av{vid}/index_{p}.html'.format(vid = vid, p = p)
 878     # Check both software
 879     logging.debug(concat_software + ', ' + download_software)
 880     # Start to find cid, api
 881     cid, partname, title, pages = find_cid_api(vid, p, cookies)
 882     #if cid is 0:
 883         #logging.warning('Cannot find cid, trying to do it brutely...')
 884         #find_cid_flvcd(videourl)
 885     if cid is 0:
 886         if IS_SLIENT == 0:
 887             logging.warning('Strange, still cannot find cid... ')
 888             is_black3 = str(raw_input('Type y for trying the unpredictable way, or input the cid by yourself; Press ENTER to quit.'))
 889         else:
 890             is_black3 = 'y'
 891         if 'y' in str(is_black3):
 892             vid = str(int(vid) - 1)
 893             p = 1
 894             find_cid_api(int(vid) - 1, p)
 895             cid = cid + 1
 896         elif str(is_black3) is '':
 897             raise NoCidException('FATAL: Cannot get cid anyway!')
 898         else:
 899             cid = str(is_black3)
 900     # start to make folders...
 901     if title is not '':
 902         folder = title
 903     else:
 904         folder = cid
 905     if len(partname) is not 0:
 906         filename = partname
 907     elif title is not '':
 908         filename = title
 909     else:
 910         filename = cid
 911     #In case cannot find which s which
 912     filename = str(p) + ' - ' + filename
 913     # In case make too much folders
 914     folder_to_make = os.getcwd() + '/' + folder
 915     if is_first_run == 0:
 916         if not os.path.exists(folder_to_make):
 917             os.makedirs(folder_to_make)
 918         is_first_run = 1
 919         os.chdir(folder_to_make)
 920     # Download Danmaku
 921     download_danmaku(cid, filename)
 922     if is_export >= 1 and IS_M3U != 1 and danmaku_only == 1:
 923         rawurl = get_video(oversea, convert_m3u = True)
 924         check_dependencies_remote_resolution('ffprobe')
 925         resolution = getvideosize(rawurl[0])[0]
 926         convert_ass(filename, probe_software, resolution = resolution)
 927     if IS_M3U == 1:
 928         rawurl = []
 929         #M3U export, then stop
 930         if oversea in {'0', '1'}:
 931             rawurl = get_video(oversea, convert_m3u = True)
 932         else:
 933             duration_list = []
 934             rawurl = get_video(oversea, convert_m3u = False)
 935             for url in rawurl:
 936                 duration_list.append(getvideosize(url)[1])
 937             rawurl = map(lambda x,y: (x, y), rawurl, duration_list)
 938         #print(rawurl)
 939         resolution = getvideosize(rawurl[0][0])[0]
 940         m3u_file = make_m3u8(rawurl)
 941         f = open(filename + '.m3u', 'w')
 942         cwd = os.getcwd()
 943         m3u_file = m3u_file.encode("utf8")
 944         f.write(m3u_file)
 945         f.close()
 946         convert_ass(filename, probe_software, resolution = resolution)
 947         logging.debug(m3u_file)
 948         raise ExportM3UException('INFO: Export to M3U')
 949     if danmaku_only == 1:
 950         raise DanmakuOnlyException('INFO: Danmaku only')
 951     # Find video location
 952     logging.info('Finding video location...')
 953     # try api
 954         # flvcd
 955     url_flag = 1
 956     rawurl = []
 957     logging.info('Trying to get download URL...')
 958     rawurl = get_video(oversea, convert_m3u = False)
 959     if len(rawurl) == 0 and oversea != '4':  # hope this never happen
 960         logging.warning('API failed, using falloff plan...')
 961         rawurl = find_link_flvcd(videourl)
 962     vid_num = len(rawurl)
 963     if IS_SLIENT == 0 and vid_num == 0:
 964         logging.warning('Cannot get download URL!')
 965         rawurl = list(str(raw_input('If you know the url, please enter it now: URL1|URL2...'))).split('|')
 966     vid_num = len(rawurl)
 967     if vid_num is 0:  # shit really hit the fan
 968         raise NoVIdeoURLException('FATAL: Cannot get video URL anyway!')
 969     logging.info('{vid_num} videos in part {part_now} to download, fetch yourself a cup of coffee...'.format(vid_num = vid_num, part_now = part_now))
 970     #Multi thread
 971     if len(rawurl) == 1:
 972         cmd = download_video_link(0,download_software,rawurl[0], thread_single_download)
 973         execute_sysencode_cmd(cmd)
 974     else:
 975         global queue
 976         queue = Queue.Queue()
 977         main_threading(download_thread, rawurl, thread_single_download)
 978         queue.join()
 979     concat_videos(concat_software, vid_num, filename)
 980     if is_export >= 1:
 981         try:
 982             convert_ass(filename, probe_software)
 983         except Exception:
 984             logging.warning('Problem with ASS conversion!')
 985             pass
 986     logging.info('Part Done!')
 987
 988 #----------------------------------------------------------------------
 989 def get_full_p(p_raw):
 990     """str->list"""
 991     p_list = []
 992     p_raw = p_raw.split(',')
 993     for item in p_raw:
 994         if '~' in item:
 995             # print(item)
 996             lower = 0
 997             higher = 0
 998             item = item.split('~')
 999             part_now = '0'
1000             try:
1001                 lower = int(item[0])
1002             except Exception:
1003                 logging.warning('Cannot read lower!')
1004             try:
1005                 higher = int(item[1])
1006             except Exception:
1007                 logging.warning('Cannot read higher!')
1008             if lower == 0 or higher == 0:
1009                 if lower == 0 and higher != 0:
1010                     lower = higher
1011                 elif lower != 0 and higher == 0:
1012                     higher = lower
1013                 else:
1014                     logging.warning('Cannot find any higher or lower, ignoring...')
1015                     # break
1016             mid = 0
1017             if higher < lower:
1018                 mid = higher
1019                 higher = lower
1020                 lower = mid
1021             p_list.append(lower)
1022             while lower < higher:
1023                 lower = lower + 1
1024                 p_list.append(lower)
1025             # break
1026         else:
1027             try:
1028                 p_list.append(int(item))
1029             except Exception:
1030                 logging.warning('Cannot read "{item}", abandon it.'.format(item = item))
1031                 # break
1032     p_list = list_del_repeat(p_list)
1033     return p_list
1034
1035 #----------------------------------------------------------------------
1036 def check_dependencies_remote_resolution(software):
1037     """"""
1038     if 'ffprobe' in software:
1039         output = commands.getstatusoutput('ffprobe --help')
1040         if str(output[0]) == '32512':
1041             FFPROBE_USABLE = 0
1042         else:
1043             FFPROBE_USABLE = 1
1044
1045 #----------------------------------------------------------------------
1046 def check_dependencies_exportm3u(IS_M3U):
1047     """int,str->int,str"""
1048     if IS_M3U == 1:
1049         output = commands.getstatusoutput('ffprobe --help')
1050         if str(output[0]) == '32512':
1051             logging.error('ffprobe DNE, python3 does not exist or not callable!')
1052             err_input = str(raw_input('Do you want to exit, ignore or stop the conversion?(e/i/s)'))
1053             if err_input == 'e':
1054                 exit()
1055             elif err_input == '2':
1056                 FFPROBE_USABLE = 0
1057             elif err_input == 's':
1058                 IS_M3U = 0
1059             else:
1060                 logging.warning('Cannot read input, stop the conversion!')
1061                 IS_M3U = 0
1062         else:
1063             FFPROBE_USABLE = 1
1064     return IS_M3U
1065
1066 #----------------------------------------------------------------------
1067 def check_dependencies_danmaku2ass(is_export):
1068     """int,str->int,str"""
1069     if is_export == 3:
1070         convert_ass = convert_ass_py3
1071         output = commands.getstatusoutput('python3 --help')
1072         if str(output[0]) == '32512' or not os.path.exists(os.path.join(LOCATION_DIR, 'danmaku2ass3.py')):
1073             logging.warning('danmaku2ass3.py DNE, python3 does not exist or not callable!')
1074             err_input = str(raw_input('Do you want to exit, use Python 2.x or stop the conversion?(e/2/s)'))
1075             if err_input == 'e':
1076                 exit()
1077             elif err_input == '2':
1078                 convert_ass = convert_ass_py2
1079                 is_export = 2
1080             elif err_input == 's':
1081                 is_export = 0
1082             else:
1083                 logging.warning('Cannot read input, stop the conversion!')
1084                 is_export = 0
1085     elif is_export == 2 or is_export == 1:
1086         convert_ass = convert_ass_py2
1087         if not os.path.exists(os.path.join(LOCATION_DIR, 'danmaku2ass2.py')):
1088             logging.warning('danmaku2ass2.py DNE!')
1089             err_input = str(raw_input('Do you want to exit, use Python 3.x or stop the conversion?(e/3/s)'))
1090             if err_input == 'e':
1091                 exit()
1092             elif err_input == '3':
1093                 convert_ass = convert_ass_py3
1094                 is_export = 3
1095             elif err_input == 's':
1096                 is_export = 0
1097             else:
1098                 logging.warning('Cannot read input, stop the conversion!')
1099                 is_export = 0
1100     else:
1101         convert_ass = convert_ass_py2
1102     return is_export, convert_ass
1103
1104 #----------------------------------------------------------------------
1105 def usage():
1106     """"""
1107     print('''
1108     Biligrab
1109
1110     https://github.com/cnbeining/Biligrab
1111     http://www.cnbeining.com/
1112
1113     Beining@ACICFG
1114
1115
1116
1117     Usage:
1118
1119     python biligrab.py (-h) (-a) (-p) (-s) (-c) (-d) (-v) (-l) (-e) (-b) (-m) (-n) (-u) (-t) (-q) (-r) (-g)
1120
1121     -h: Default: None
1122         Print this usage file.
1123
1124     -a: Default: None
1125         The av number.
1126         If not set, Biligrab will use the fallback interactive mode.
1127         Support "~", "," and mix use.
1128         Examples:
1129             Input        Output
1130               1           [1]
1131              1,2         [1, 2]
1132              1~3        [1, 2, 3]
1133             1,2~3       [1, 2, 3]
1134
1135     -p: Default: 0
1136         The part number.
1137         Able to use the same syntax as "-a".
1138         If set to 0, Biligrab will download all the available parts in the video.
1139
1140     -s: Default: 0
1141     Source to download.
1142     0: The original API source, can be Letv backup,
1143        and can fail if the original video is not available(e.g., deleted)
1144     1: The CDN API source, "oversea accelerate".
1145        Can be MINICDN backup in Mainland China or oversea.
1146        Good to bypass some bangumi's restrictions.
1147     2: Force to use the original source.
1148        Use Flvcd to parse the video, but would fail if
1149        1) The original source DNE, e.g., some old videos
1150        2) The original source is Letvcloud itself.
1151        3) Other unknown reason(s) that stops Flvcd from parsing the video.
1152     For any video that failed to parse, Biligrab will try to use Flvcd.
1153     (Mainly for oversea users regarding to copyright-restricted bangumies.)
1154     If the API is blocked, Biligrab would fake the UA.
1155     3: (Not stable) Use the HTML5 API.
1156        This works for downloading some cached Letvcloud videos, but is slow, and would fail for no reason sometimes.
1157        Will retry if unavailable.
1158     4: Use Flvcd.
1159        Good to fight with oversea and copyright restriction, but not working with iQiyi.
1160        May retrieve better quality video, especially for Youku.
1161     5: Use BilibiliPr.
1162        Good to fight with some copyright restriction that BilibiliPr can fix.
1163        Not always working though.
1164     6: Use You-get (https://github.com/soimort/you-get).
1165        You need a you-get callable directly like "you-get -u blahblah".
1166
1167     -c: Default: ./bilicookies
1168     The path of cookies.
1169     Use cookies to visit member-only videos.
1170
1171     -d: Default: None
1172     Set the desired download software.
1173     Biligrab supports aria2c(16 threads), axel(20 threads), wget and curl by far.
1174     If not set, Biligrab will detect an available one;
1175     If none of those is available, Biligrab will quit.
1176     For more software support, please open an issue at https://github.com/cnbeining/Biligrab/issues/
1177
1178     -v: Default:None
1179     Set the desired concatenate software.
1180     Biligrab supports ffmpeg by far.
1181     If not set, Biligrab will detect an available one;
1182     If none of those is available, Biligrab will quit.
1183     For more software support, please open an issue at https://github.com/cnbeining/Biligrab/issues/
1184     Make sure you include a *working* command line example of this software!
1185
1186     -l: Default: INFO
1187     Dump the log of the output for better debugging.
1188     Can be set to debug.
1189
1190     -e: Default: 1
1191     Export Danmaku to ASS file.
1192     Fulfilled with danmaku2ass(https://github.com/m13253/danmaku2ass/tree/py2),
1193     Author: @m13253, GPLv3 License.
1194     *For issue with this function, if you think the problem lies on the danmaku2ass side,
1195     please open the issue at both projects.*
1196     If set to 1 or 2, Biligrab will use Danmaku2ass's py2 branch.
1197     If set to 3, Biligrab will use Danmaku2ass's master branch, which would require
1198     a python3 callable via 'python3'.
1199     If python3 not callable or danmaku2ass2/3 DNE, Biligrab will ask for action.
1200
1201     -b: Default: None
1202     Set the probe software.
1203     Biligrab supports Mediainfo and FFprobe.
1204     If not set, Biligrab will detect an available one;
1205     If none of those is available, Biligrab will quit.
1206     For more software support, please open an issue at https://github.com/cnbeining/Biligrab/issues/
1207     Make sure you include a *working* command line example of this software!
1208
1209     -m: Default: 0
1210     Only download the danmaku.
1211
1212     -n: Default: 0
1213     Silent Mode.
1214     Biligrab will not ask any question.
1215
1216     -u: Default: 0
1217     Export video link to .m3u file, which can be used with MPlayer, mpc, VLC, etc.
1218     Biligrab will export a m3u8 instead of downloading any video(s).
1219     Can be broken with sources other than 0 or 1.
1220
1221     -t: Default: None
1222     The number of Mylist.
1223     Biligrab will process all the videos in this list.
1224
1225     -q: Default: 3
1226     The thread number for downloading.
1227     Good to fix overhead problem.
1228
1229     -r: Default: -1
1230     Select video quality.
1231     Only works with Source 0 or 1.
1232     Range: 0~4, higher for better quality.
1233
1234     -g: Default: 6
1235     Threads for downloading every part.
1236     Works with aria2 and axel.
1237
1238     -i: Default: None
1239     Fake IP address.
1240     ''')
1241
1242 #----------------------------------------------------------------------
1243 if __name__ == '__main__':
1244     is_first_run, is_export, danmaku_only, IS_SLIENT, IS_M3U, mylist, time_fetch, download_thread, QUALITY, thread_single_download = 0, 1, 0, 0, 0, 0, 5, 16, -1, 16
1245     argv_list,av_list = [], []
1246     argv_list = sys.argv[1:]
1247     p_raw, vid, oversea, cookiepath, download_software, concat_software, probe_software, vid_raw, LOG_LEVEL, FAKE_IP, IS_FAKE_IP = '', '', '', '', '', '', '', '', 'INFO', '', 0
1248     convert_ass = convert_ass_py2
1249     try:
1250         opts, args = getopt.getopt(argv_list, "ha:p:s:c:d:v:l:e:b:m:n:u:t:q:r:g:i:",
1251                                    ['help', "av=", 'part=', 'source=', 'cookie=', 'download=', 'concat=', 'log=', 'export=', 'probe=', 'danmaku=', 'slient=', 'm3u=', 'mylist=', 'thread=', 'quality=', 'thread_single=', 'fake-ip='])
1252     except getopt.GetoptError:
1253         usage()
1254         exit()
1255     for o, a in opts:
1256         if o in ('-h', '--help'):
1257             usage()
1258             exit()
1259         if o in ('-a', '--av'):
1260             vid_raw = a
1261         if o in ('-p', '--part'):
1262             p_raw = a
1263         if o in ('-s', '--source'):
1264             oversea = a
1265         if o in ('-c', '--cookie'):
1266             cookiepath = a
1267             if cookiepath == '':
1268                 logging.warning('No cookie path set, use default: ./bilicookies')
1269                 cookiepath = './bilicookies'
1270         if o in ('-d', '--download'):
1271             download_software = a
1272         if o in ('-v', '--concat'):
1273             concat_software = a
1274         if o in ('-l', '--log'):
1275             try:
1276                 LOG_LEVEL = str(a)
1277             except Exception:
1278                 LOG_LEVEL = 'INFO'
1279         if o in ('-e', '--export'):
1280             is_export = int(a)
1281         if o in ('-b', '--probe'):
1282             probe_software = a
1283         if o in ('-m', '--danmaku'):
1284             danmaku_only = int(a)
1285         if o in ('-n', '--slient'):
1286             IS_SLIENT = int(a)
1287         if o in ('-u', '--m3u'):
1288             IS_M3U = int(a)
1289         if o in ('-t', '--mylist'):
1290             mylist = a
1291         if o in ('-q', '--thread'):
1292             download_thread = int(a)
1293         if o in ('-r', '--quality'):
1294             QUALITY = int(a)
1295         if o in ('-g', '--thread_single'):
1296             thread_single_download = int(a)
1297         if o in ('-i', '--fake-ip'):
1298             FAKE_IP = a
1299             IS_FAKE_IP = 1
1300     if len(vid_raw) == 0:
1301         vid_raw = str(raw_input('av'))
1302         p_raw = str(raw_input('P'))
1303         oversea = str(raw_input('Source?'))
1304         cookiepath = './bilicookies'
1305     logging.basicConfig(level = logging_level_reader(LOG_LEVEL))
1306     logging.debug('FAKE IP: ' + str(IS_FAKE_IP) + ' ' + FAKE_IP)
1307     av_list = get_full_p(vid_raw)
1308     if mylist != 0:
1309         av_list += mylist_to_aid_list(mylist)
1310     logging.debug('av_list')
1311     if len(cookiepath) == 0:
1312         cookiepath = './bilicookies'
1313     if len(p_raw) == 0:
1314         logging.info('No part number set, download all the parts.')
1315         p_raw = '0'
1316     if len(oversea) == 0:
1317         oversea = '0'
1318         logging.info('Oversea not set, use original API(methon 0).')
1319     IS_M3U = check_dependencies_exportm3u(IS_M3U)
1320     if IS_M3U == 1 and oversea not in {'0', '1'}:
1321         # See issue #8
1322         logging.info('M3U exporting with source other than 0 or 1 can be broken, and lead to wrong duration!')
1323         if IS_SLIENT == 0:
1324             input_raw = str(raw_input('Enter "q" to quit, or enter the source you want.'))
1325             if input_raw == 'q':
1326                 exit()
1327             else:
1328                 oversea = input_raw
1329     concat_software, download_software, probe_software = check_dependencies(download_software, concat_software, probe_software)
1330     p_list = get_full_p(p_raw)
1331     if len(av_list) > 1 and len(p_list) > 1:
1332         logging.warning('You are downloading multi parts from multiple videos! This may result in unpredictable outputs!')
1333         if IS_SLIENT == 0:
1334             input_raw = str(raw_input('Enter "y" to continue, "n" to only download the first part, "q" to quit, or enter the part number you want.'))
1335             if input_raw == 'y':
1336                 pass
1337             elif input_raw == 'n':
1338                 p_list = ['1']
1339             elif input_raw == 'q':
1340                 exit()
1341             else:
1342                 p_list = get_full_p(input_raw)
1343     cookies = read_cookie(cookiepath)
1344     global BILIGRAB_HEADER, BILIGRAB_UA
1345     # deal with danmaku2ass's drama / Twice in case someone failed to check dependencies
1346     is_export, convert_ass = check_dependencies_danmaku2ass(is_export)
1347     is_export, convert_ass = check_dependencies_danmaku2ass(is_export)
1348     python_ver_str = '.'.join([str(i) for i in sys.version_info[:2]])
1349     BILIGRAB_UA = 'Biligrab/{VER} (cnbeining@gmail.com) (Python-urllib/{python_ver_str}, like libcurl/1.0 NSS-Mozilla/2.0)'.format(VER = VER, python_ver_str = python_ver_str)
1350
1351     #BILIGRAB_UA = 'Biligrab / ' + str(VER) + ' (cnbeining@gmail.com) (like )'
1352     BILIGRAB_HEADER = {'User-Agent': BILIGRAB_UA, 'Cache-Control': 'no-cache', 'Pragma': 'no-cache', 'Cookie': cookies[0]}
1353     if LOG_LEVEL == 'DEBUG':
1354         logging.debug('!!!!!!!!!!!!!!!!!!!!!!!\nWARNING: This log contains some sensitive data. You may want to delete some part of the data before you post it publicly!\n!!!!!!!!!!!!!!!!!!!!!!!')
1355         logging.debug('BILIGRAB_HEADER')
1356         try:
1357             request = urllib2.Request('http://ipinfo.io/json', headers=FAKE_HEADER)
1358             response = urllib2.urlopen(request)
1359             data = response.read()
1360             print('!!!!!!!!!!!!!!!!!!!!!!!\nWARNING: This log contains some sensitive data. You may want to delete some part of the data before you post it publicly!\n!!!!!!!!!!!!!!!!!!!!!!!')
1361             print('=======================DUMP DATA==================')
1362             print(data)
1363             print('========================DATA END==================')
1364             print('DEBUG: ' + str(av_list))
1365         except Exception:
1366             print('WARNING: Cannot connect to IP-geo database server!')
1367             pass
1368     for av in av_list:
1369         vid = str(av)
1370         if str(p_raw) == '0':
1371             logging.info('You are downloading all the parts in this video...')
1372             try:
1373                 p_raw = str('1~' + find_cid_api(vid, p_raw, cookies)[3])
1374                 p_list = get_full_p(p_raw)
1375             except Exception:
1376                 logging.info('Error when reading all the parts!')
1377                 if IS_SLIENT == 0:
1378                     input_raw = str(raw_input('Enter the part number you want, or "q" to quit.'))
1379                     if input_raw == '0':
1380                         print('ERROR: Cannot use all the parts!')
1381                         exit()
1382                     elif input_raw == 'q':
1383                         exit()
1384                     else:
1385                         p_list = get_full_p(input_raw)
1386                 else:
1387                     logging.info('Download the first part of the video...')
1388                     p_raw = '1'
1389                     p_list = [1]
1390             logging.info('Your target download is av{vid}, part {p_raw}, from source {oversea}'.format(vid = vid, p_raw = p_raw, oversea = oversea))
1391         for p in p_list:
1392             reload(sys)
1393             sys.setdefaultencoding('utf-8')
1394             part_now = str(p)
1395             try:
1396                 logging.info('Downloading part {p} ...'.format(p = p))
1397                 main(vid, p, oversea, cookies, download_software, concat_software, is_export, probe_software, danmaku_only, time_fetch, download_thread, thread_single_download)
1398             except DanmakuOnlyException:
1399                 pass
1400             except ExportM3UException:
1401                 pass
1402             except Exception as e:
1403                 print('ERROR: Biligrab failed: %s' % e)
1404                 print('       If you think this should not happen, please dump your log using "-l", and open a issue at https://github.com/cnbeining/Biligrab/issues .')
1405                 print('       Make sure you delete all the sensitive data before you post it publicly.')
1406                 traceback.print_exc()
1407     exit()