plugins/togo/togo.py

   1 import cookielib
   2 import logging
   3 import os
   4 import subprocess
   5 import thread
   6 import time
   7 import urllib2
   8 import urlparse
   9 from urllib import quote, unquote
  10 from xml.dom import minidom
  11 from xml.sax.saxutils import escape
  12
  13 from Cheetah.Template import Template
  14
  15 import config
  16 import metadata
  17 from plugin import EncodeUnicode, Plugin
  18
  19 logger = logging.getLogger('pyTivo.togo')
  20 tag_data = metadata.tag_data
  21
  22 SCRIPTDIR = os.path.dirname(__file__)
  23
  24 CLASS_NAME = 'ToGo'
  25
  26 # Characters to remove from filenames
  27
  28 BADCHAR = {'\\': '-', '/': '-', ':': ' -', ';': ',', '*': '.',
  29            '?': '.', '!': '.', '"': "'", '<': '(', '>': ')', '|': ' '}
  30
  31 # Some error/status message templates
  32
  33 MISSING = """<h3>Missing Data</h3> <p>You must set both "tivo_mak" and
  34 "togo_path" before using this function.</p>"""
  35
  36 TRANS_QUEUE = """<h3>Queued for Transfer</h3> <p>%s</p> <p>queued for
  37 transfer to:</p> <p>%s</p>"""
  38
  39 TRANS_STOP = """<h3>Transfer Stopped</h3> <p>Your transfer of:</p>
  40 <p>%s</p> <p>has been stopped.</p>"""
  41
  42 UNQUEUE = """<h3>Removed from Queue</h3> <p>%s</p> <p>has been removed
  43 from the queue.</p>"""
  44
  45 UNABLE = """<h3>Unable to Connect to TiVo</h3> <p>pyTivo was unable to
  46 connect to the TiVo at %s.</p> <p>This is most likely caused by an
  47 incorrect Media Access Key. Please return to the Settings page and
  48 double check your <b>tivo_mak</b> setting.</p> <pre>%s</pre>"""
  49
  50 # Preload the templates
  51 tnname = os.path.join(SCRIPTDIR, 'templates', 'npl.tmpl')
  52 NPL_TEMPLATE = file(tnname, 'rb').read()
  53
  54 status = {} # Global variable to control download threads
  55 tivo_cache = {} # Cache of TiVo NPL
  56 queue = {} # Recordings to download -- list per TiVo
  57 basic_meta = {} # Data from NPL, parsed, indexed by progam URL
  58
  59 def null_cookie(name, value):
  60     return cookielib.Cookie(0, name, value, None, False, '', False,
  61         False, '', False, False, None, False, None, None, None)
  62
  63 auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
  64 cj = cookielib.CookieJar()
  65 cj.set_cookie(null_cookie('sid', 'ADEADDA7EDEBAC1E'))
  66 tivo_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
  67                                    urllib2.HTTPBasicAuthHandler(auth_handler),
  68                                    urllib2.HTTPDigestAuthHandler(auth_handler))
  69
  70 class ToGo(Plugin):
  71     CONTENT_TYPE = 'text/html'
  72
  73     def tivo_open(self, url):
  74         # Loop just in case we get a server busy message
  75         while True:
  76             try:
  77                 # Open the URL using our authentication/cookie opener
  78                 return tivo_opener.open(url)
  79
  80             # Do a retry if the TiVo responds that the server is busy
  81             except urllib2.HTTPError, e:
  82                 if e.code == 503:
  83                     time.sleep(5)
  84                     continue
  85
  86                 # Log and throw the error otherwise
  87                 logger.error(e)
  88                 raise
  89
  90     def NPL(self, handler, query):
  91
  92         def getint(thing):
  93             try:
  94                 result = int(thing)
  95             except:
  96                 result = 0
  97             return result
  98
  99         global basic_meta
 100         shows_per_page = 50 # Change this to alter the number of shows returned
 101         folder = ''
 102         FirstAnchor = ''
 103         has_tivodecode = bool(config.get_bin('tivodecode'))
 104
 105         if 'TiVo' in query:
 106             tivoIP = query['TiVo'][0]
 107             tsn = config.tivos_by_ip(tivoIP)
 108             tivo_name = config.tivos[tsn].get('name', tivoIP)
 109             tivo_mak = config.get_tsn('tivo_mak', tsn)
 110             if 'port' in config.tivos[tsn]:
 111                 ip_port = '%s:%d' % (tivoIP, config.tivos[tsn]['port'])
 112             else:
 113                 ip_port = tivoIP
 114             theurl = ('https://' + ip_port +
 115                       '/TiVoConnect?Command=QueryContainer&ItemCount=' +
 116                       str(shows_per_page) + '&Container=/NowPlaying')
 117             if 'Folder' in query:
 118                 folder += query['Folder'][0]
 119                 theurl += '/' + folder
 120             if 'AnchorItem' in query:
 121                 theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
 122             if 'AnchorOffset' in query:
 123                 theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
 124
 125             if (theurl not in tivo_cache or
 126                 (time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
 127                 # if page is not cached or old then retreive it
 128                 auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
 129                 try:
 130                     page = self.tivo_open(theurl)
 131                 except IOError, e:
 132                     handler.redir(UNABLE % (tivoIP, e), 10)
 133                     return
 134                 tivo_cache[theurl] = {'thepage': minidom.parse(page),
 135                                       'thepage_time': time.time()}
 136                 page.close()
 137
 138             xmldoc = tivo_cache[theurl]['thepage']
 139             items = xmldoc.getElementsByTagName('Item')
 140             TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
 141             ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
 142             ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
 143             title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
 144             if items:
 145                 FirstAnchor = tag_data(items[0], 'Links/Content/Url')
 146
 147             data = []
 148             for item in items:
 149                 entry = {}
 150                 entry['ContentType'] = tag_data(item, 'Details/ContentType')
 151                 for tag in ('CopyProtected', 'UniqueId'):
 152                     value = tag_data(item, 'Details/' + tag)
 153                     if value:
 154                         entry[tag] = value
 155                 if entry['ContentType'] == 'x-tivo-container/folder':
 156                     entry['Title'] = tag_data(item, 'Details/Title')
 157                     entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
 158                     lc = tag_data(item, 'Details/LastCaptureDate')
 159                     if not lc:
 160                         lc = tag_data(item, 'Details/LastChangeDate')
 161                     entry['LastChangeDate'] = time.strftime('%b %d, %Y',
 162                         time.localtime(int(lc, 16)))
 163                 else:
 164                     keys = {'Icon': 'Links/CustomIcon/Url',
 165                             'Url': 'Links/Content/Url',
 166                             'Details': 'Links/TiVoVideoDetails/Url',
 167                             'SourceSize': 'Details/SourceSize',
 168                             'Duration': 'Details/Duration',
 169                             'CaptureDate': 'Details/CaptureDate'}
 170                     for key in keys:
 171                         value = tag_data(item, keys[key])
 172                         if value:
 173                             entry[key] = value
 174
 175                     rawsize = entry['SourceSize']
 176                     entry['SourceSize'] = metadata.human_size(rawsize)
 177
 178                     if 'Duration' in entry:
 179                         dur = getint(entry['Duration']) / 1000
 180                         entry['Duration'] = ( '%d:%02d:%02d' %
 181                             (dur / 3600, (dur % 3600) / 60, dur % 60) )
 182
 183                     entry['CaptureDate'] = time.strftime('%b %d, %Y',
 184                         time.localtime(int(entry['CaptureDate'], 16)))
 185
 186                     url = entry['Url']
 187                     if url in basic_meta:
 188                         entry.update(basic_meta[url])
 189                     else:
 190                         basic_data = metadata.from_container(item)
 191                         entry.update(basic_data)
 192                         basic_meta[url] = basic_data
 193
 194                 data.append(entry)
 195         else:
 196             data = []
 197             tivoIP = ''
 198             TotalItems = 0
 199             ItemStart = 0
 200             ItemCount = 0
 201             title = ''
 202
 203         t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
 204         t.escape = escape
 205         t.quote = quote
 206         t.folder = folder
 207         t.status = status
 208         if tivoIP in queue:
 209             t.queue = queue[tivoIP]
 210         t.has_tivodecode = has_tivodecode
 211         t.togo_mpegts = config.is_ts_capable(tsn)
 212         t.tname = tivo_name
 213         t.tivoIP = tivoIP
 214         t.container = handler.cname
 215         t.data = data
 216         t.len = len
 217         t.TotalItems = getint(TotalItems)
 218         t.ItemStart = getint(ItemStart)
 219         t.ItemCount = getint(ItemCount)
 220         t.FirstAnchor = quote(FirstAnchor)
 221         t.shows_per_page = shows_per_page
 222         t.title = title
 223         handler.send_html(str(t), refresh='300')
 224
 225     def get_tivo_file(self, tivoIP, url, mak, togo_path):
 226         # global status
 227         status[url].update({'running': True, 'queued': False})
 228
 229         parse_url = urlparse.urlparse(url)
 230
 231         name = unquote(parse_url[2])[10:].split('.')
 232         try:
 233             id = unquote(parse_url[4]).split('id=')[1]
 234             name.insert(-1, ' - ' + id)
 235         except:
 236             pass
 237         ts = status[url]['ts_format']
 238         if status[url]['decode']:
 239             if ts:
 240                 name[-1] = 'ts'
 241             else:
 242                 name[-1] = 'mpg'
 243         else:
 244             if ts:
 245                 name.insert(-1, ' (TS)')
 246             else:
 247                 name.insert(-1, ' (PS)')
 248         name.insert(-1, '.')
 249         name = ''.join(name)
 250         for ch in BADCHAR:
 251             name = name.replace(ch, BADCHAR[ch])
 252         outfile = os.path.join(togo_path, name)
 253
 254         if status[url]['save']:
 255             meta = basic_meta[url]
 256             details_url = 'https://%s/TiVoVideoDetails?id=%s' % (tivoIP, id)
 257             try:
 258                 handle = self.tivo_open(details_url)
 259                 meta.update(metadata.from_details(handle.read()))
 260                 handle.close()
 261             except:
 262                 pass
 263             metafile = open(outfile + '.txt', 'w')
 264             metadata.dump(metafile, meta)
 265             metafile.close()
 266
 267         auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
 268         try:
 269             if status[url]['ts_format']:
 270                 handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
 271             else:
 272                 handle = self.tivo_open(url)
 273         except Exception, msg:
 274             status[url]['running'] = False
 275             status[url]['error'] = str(msg)
 276             return
 277
 278         tivo_name = config.tivos[config.tivos_by_ip(tivoIP)].get('name', tivoIP)
 279
 280         logger.info('[%s] Start getting "%s" from %s' %
 281                     (time.strftime('%d/%b/%Y %H:%M:%S'), outfile, tivo_name))
 282
 283         if status[url]['decode']:
 284             tivodecode_path = config.get_bin('tivodecode')
 285             tcmd = [tivodecode_path, '-m', mak, '-o', outfile, '-']
 286             tivodecode = subprocess.Popen(tcmd, stdin=subprocess.PIPE,
 287                                           bufsize=(512 * 1024))
 288             f = tivodecode.stdin
 289         else:
 290             f = open(outfile, 'wb')
 291         length = 0
 292         start_time = time.time()
 293         last_interval = start_time
 294         now = start_time
 295         try:
 296             while status[url]['running']:
 297                 output = handle.read(1024000)
 298                 if not output:
 299                     break
 300                 length += len(output)
 301                 f.write(output)
 302                 now = time.time()
 303                 elapsed = now - last_interval
 304                 if elapsed >= 5:
 305                     status[url]['rate'] = '%.2f Mb/s' % (length * 8.0 /
 306                         (elapsed * 1024 * 1024))
 307                     status[url]['size'] += length
 308                     length = 0
 309                     last_interval = now
 310             if status[url]['running']:
 311                 status[url]['finished'] = True
 312         except Exception, msg:
 313             status[url]['running'] = False
 314             logger.info(msg)
 315         handle.close()
 316         f.close()
 317         status[url]['size'] += length
 318         if status[url]['running']:
 319             mega_elapsed = (now - start_time) * 1024 * 1024
 320             if mega_elapsed < 1:
 321                 mega_elapsed = 1
 322             size = status[url]['size']
 323             rate = size * 8.0 / mega_elapsed
 324             logger.info('[%s] Done getting "%s" from %s, %d bytes, %.2f Mb/s' %
 325                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 326                          tivo_name, size, rate))
 327             status[url]['running'] = False
 328         else:
 329             os.remove(outfile)
 330             if status[url]['save']:
 331                 os.remove(outfile + '.txt')
 332             logger.info('[%s] Transfer of "%s" from %s aborted' %
 333                         (time.strftime('%d/%b/%Y %H:%M:%S'), outfile,
 334                          tivo_name))
 335             del status[url]
 336
 337     def process_queue(self, tivoIP, mak, togo_path):
 338         while queue[tivoIP]:
 339             time.sleep(5)
 340             url = queue[tivoIP][0]
 341             self.get_tivo_file(tivoIP, url, mak, togo_path)
 342             queue[tivoIP].pop(0)
 343         del queue[tivoIP]
 344
 345     def ToGo(self, handler, query):
 346         togo_path = config.get_server('togo_path')
 347         for name, data in config.getShares():
 348             if togo_path == name:
 349                 togo_path = data.get('path')
 350         if togo_path:
 351             tivoIP = query['TiVo'][0]
 352             tsn = config.tivos_by_ip(tivoIP)
 353             tivo_mak = config.get_tsn('tivo_mak', tsn)
 354             urls = query.get('Url', [])
 355             decode = 'decode' in query
 356             save = 'save' in query
 357             ts_format = 'ts_format' in query
 358             for theurl in urls:
 359                 status[theurl] = {'running': False, 'error': '', 'rate': '',
 360                                   'queued': True, 'size': 0, 'finished': False,
 361                                   'decode': decode, 'save': save,
 362                                   'ts_format': ts_format}
 363                 if tivoIP in queue:
 364                     queue[tivoIP].append(theurl)
 365                 else:
 366                     queue[tivoIP] = [theurl]
 367                     thread.start_new_thread(ToGo.process_queue,
 368                                             (self, tivoIP, tivo_mak, togo_path))
 369                 logger.info('[%s] Queued "%s" for transfer to %s' %
 370                             (time.strftime('%d/%b/%Y %H:%M:%S'),
 371                              unquote(theurl), togo_path))
 372             urlstring = '<br>'.join([unquote(x) for x in urls])
 373             message = TRANS_QUEUE % (urlstring, togo_path)
 374         else:
 375             message = MISSING
 376         handler.redir(message, 5)
 377
 378     def ToGoStop(self, handler, query):
 379         theurl = query['Url'][0]
 380         status[theurl]['running'] = False
 381         handler.redir(TRANS_STOP % unquote(theurl))
 382
 383     def Unqueue(self, handler, query):
 384         theurl = query['Url'][0]
 385         tivoIP = query['TiVo'][0]
 386         del status[theurl]
 387         queue[tivoIP].remove(theurl)
 388         logger.info('[%s] Removed "%s" from queue' %
 389                     (time.strftime('%d/%b/%Y %H:%M:%S'),
 390                      unquote(theurl)))
 391         handler.redir(UNQUEUE % unquote(theurl))