plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 LIKELYTS = """.ts .tp .trp .3g2 .3gp .3gp2 .3gpp .m2t .m2ts .mts .mp4
  49 .m4v .flv .mkv .mov .wtv .dvr-ms .webm""".split()
  50
  51 use_extensions = True
  52 try:
  53     assert(config.get_bin('ffmpeg'))
  54 except:
  55     use_extensions = False
  56
  57 queue = []  # Recordings to push
  58
  59 def uniso(iso):
  60     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  61
  62 def isodt(iso):
  63     return datetime(*uniso(iso)[:6])
  64
  65 def isogm(iso):
  66     return int(calendar.timegm(uniso(iso)))
  67
  68 class Pushable(object):
  69
  70     def push_one_file(self, f):
  71         file_info = VideoDetails()
  72         file_info['valid'] = transcode.supported_format(f['path'])
  73
  74         mime = 'video/mpeg'
  75         if config.isHDtivo(f['tsn']):
  76             for m in ['video/mp4', 'video/bif']:
  77                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  78                     mime = m
  79                     break
  80
  81             if (mime == 'video/mpeg' and
  82                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  83                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  84                 if new_path:
  85                     mime = 'video/mp4'
  86                     f['name'] = new_path
  87
  88         if file_info['valid']:
  89             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  90
  91         url = f['url'] + quote(f['name'])
  92
  93         title = file_info['seriesTitle']
  94         if not title:
  95             title = file_info['title']
  96
  97         source = file_info['seriesId']
  98         if not source:
  99             source = title
 100
 101         subtitle = file_info['episodeTitle']
 102         try:
 103             m = mind.getMind(f['tsn'])
 104             m.pushVideo(
 105                 tsn = f['tsn'],
 106                 url = url,
 107                 description = file_info['description'],
 108                 duration = file_info['duration'] / 1000,
 109                 size = file_info['size'],
 110                 title = title,
 111                 subtitle = subtitle,
 112                 source = source,
 113                 mime = mime,
 114                 tvrating = file_info['tvRating'])
 115         except Exception, msg:
 116             logger.error(msg)
 117
 118     def process_queue(self):
 119         while queue:
 120             time.sleep(5)
 121             item = queue.pop(0)
 122             self.push_one_file(item)
 123
 124     def readip(self):
 125         """ returns your external IP address by querying dyndns.org """
 126         f = urllib.urlopen('http://checkip.dyndns.org/')
 127         s = f.read()
 128         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 129         return m.group(0)
 130
 131     def Push(self, handler, query):
 132         try:
 133             tsn = query['tsn'][0]
 134         except:
 135             logger.error('Push requires a TiVo Service Number')
 136             handler.send_error(404)
 137             return
 138
 139         if not tsn in config.tivos:
 140             for key, value in config.tivos.items():
 141                 if value.get('name') == tsn:
 142                     tsn = key
 143                     break
 144         try:
 145             tivo_name = config.tivos[tsn]['name']
 146         except:
 147             tivo_name = tsn
 148
 149         container = quote(query['Container'][0].split('/')[0])
 150         ip = config.get_ip(tsn)
 151         port = config.getPort()
 152
 153         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 154         if config.getIsExternal(tsn):
 155             exturl = config.get_server('externalurl')
 156             if exturl:
 157                 if not exturl.endswith('/'):
 158                     exturl += '/'
 159                 baseurl = exturl + container
 160             else:
 161                 ip = self.readip()
 162                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 163
 164         path = self.get_local_base_path(handler, query)
 165
 166         files = query.get('File', [])
 167         for f in files:
 168             file_path = os.path.normpath(path + '/' + f)
 169             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 170                           'url': baseurl})
 171             if len(queue) == 1:
 172                 thread.start_new_thread(Video.process_queue, (self,))
 173
 174             logger.info('[%s] Queued "%s" for Push to %s' %
 175                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 176                          unicode(file_path, 'utf-8'), tivo_name))
 177
 178         files = [unicode(f, 'utf-8') for f in files]
 179         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 180
 181 class BaseVideo(Plugin):
 182
 183     CONTENT_TYPE = 'x-container/tivo-videos'
 184
 185     tvbus_cache = LRUCache(1)
 186
 187     def video_file_filter(self, full_path, type=None):
 188         if os.path.isdir(unicode(full_path, 'utf-8')):
 189             return True
 190         if use_extensions:
 191             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 192         else:
 193             return transcode.supported_format(full_path)
 194
 195     def send_file(self, handler, path, query):
 196         mime = 'video/x-tivo-mpeg'
 197         tsn = handler.headers.getheader('tsn', '')
 198         try:
 199             assert(tsn)
 200             tivo_name = config.tivos[tsn].get('name', tsn)
 201         except:
 202             tivo_name = handler.address_string()
 203
 204         is_tivo_file = (path[-5:].lower() == '.tivo')
 205
 206         if 'Format' in query:
 207             mime = query['Format'][0]
 208
 209         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 210         compatible = (not needs_tivodecode and
 211                       transcode.tivo_compatible(path, tsn, mime)[0])
 212
 213         try:  # "bytes=XXX-"
 214             offset = int(handler.headers.getheader('Range')[6:-1])
 215         except:
 216             offset = 0
 217
 218         if needs_tivodecode:
 219             valid = bool(config.get_bin('tivodecode') and
 220                          config.get_server('tivo_mak'))
 221         else:
 222             valid = True
 223
 224         if valid and offset:
 225             valid = ((compatible and offset < os.path.getsize(path)) or
 226                      (not compatible and transcode.is_resumable(path, offset)))
 227
 228         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 229         faking = (mime == 'video/x-tivo-mpeg' and
 230                   not (is_tivo_file and compatible))
 231         fname = unicode(path, 'utf-8')
 232         thead = ''
 233         if faking:
 234             thead = self.tivo_header(tsn, path, mime)
 235         if compatible:
 236             size = os.path.getsize(fname) + len(thead)
 237             handler.send_response(200)
 238             handler.send_header('Content-Length', size - offset)
 239             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 240                                 (offset, size - offset - 1, size))
 241         else:
 242             handler.send_response(206)
 243             handler.send_header('Transfer-Encoding', 'chunked')
 244         handler.send_header('Content-Type', mime)
 245         handler.end_headers()
 246
 247         logger.info('[%s] Start sending "%s" to %s' %
 248                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 249         start = time.time()
 250         count = 0
 251
 252         if valid:
 253             if compatible:
 254                 if faking and not offset:
 255                     handler.wfile.write(thead)
 256                 logger.debug('"%s" is tivo compatible' % fname)
 257                 f = open(fname, 'rb')
 258                 try:
 259                     if mime == 'video/mp4':
 260                         count = qtfaststart.process(f, handler.wfile, offset)
 261                     else:
 262                         if offset:
 263                             offset -= len(thead)
 264                             f.seek(offset)
 265                         while True:
 266                             block = f.read(512 * 1024)
 267                             if not block:
 268                                 break
 269                             handler.wfile.write(block)
 270                             count += len(block)
 271                 except Exception, msg:
 272                     logger.info(msg)
 273                 f.close()
 274             else:
 275                 logger.debug('"%s" is not tivo compatible' % fname)
 276                 if offset:
 277                     count = transcode.resume_transfer(path, handler.wfile,
 278                                                       offset)
 279                 else:
 280                     count = transcode.transcode(False, path, handler.wfile,
 281                                                 tsn, mime, thead)
 282         try:
 283             if not compatible:
 284                  handler.wfile.write('0\r\n\r\n')
 285             handler.wfile.flush()
 286         except Exception, msg:
 287             logger.info(msg)
 288
 289         mega_elapsed = (time.time() - start) * 1024 * 1024
 290         if mega_elapsed < 1:
 291             mega_elapsed = 1
 292         rate = count * 8.0 / mega_elapsed
 293         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 294                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 295                      tivo_name, count, rate))
 296
 297         if fname.endswith('.pyTivo-temp'):
 298             os.remove(fname)
 299
 300     def __duration(self, full_path):
 301         return transcode.video_info(full_path)['millisecs']
 302
 303     def __total_items(self, full_path):
 304         count = 0
 305         try:
 306             full_path = unicode(full_path, 'utf-8')
 307             for f in os.listdir(full_path):
 308                 if f.startswith('.'):
 309                     continue
 310                 f = os.path.join(full_path, f)
 311                 f2 = f.encode('utf-8')
 312                 if os.path.isdir(f):
 313                     count += 1
 314                 elif use_extensions:
 315                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 316                         count += 1
 317                 elif f2 in transcode.info_cache:
 318                     if transcode.supported_format(f2):
 319                         count += 1
 320         except:
 321             pass
 322         return count
 323
 324     def __est_size(self, full_path, tsn='', mime=''):
 325         # Size is estimated by taking audio and video bit rate adding 2%
 326
 327         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 328             return os.path.getsize(unicode(full_path, 'utf-8'))
 329         else:
 330             # Must be re-encoded
 331             audioBPS = config.getMaxAudioBR(tsn) * 1000
 332             #audioBPS = config.strtod(config.getAudioBR(tsn))
 333             videoBPS = transcode.select_videostr(full_path, tsn)
 334             bitrate =  audioBPS + videoBPS
 335             return int((self.__duration(full_path) / 1000) *
 336                        (bitrate * 1.02 / 8))
 337
 338     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 339         data = {}
 340         vInfo = transcode.video_info(full_path)
 341
 342         if ((int(vInfo['vHeight']) >= 720 and
 343              config.getTivoHeight >= 720) or
 344             (int(vInfo['vWidth']) >= 1280 and
 345              config.getTivoWidth >= 1280)):
 346             data['showingBits'] = '4096'
 347
 348         data.update(metadata.basic(full_path, mtime))
 349         if full_path[-5:].lower() == '.tivo':
 350             data.update(metadata.from_tivo(full_path))
 351         if full_path[-4:].lower() == '.wtv':
 352             data.update(metadata.from_mscore(vInfo['rawmeta']))
 353
 354         if 'episodeNumber' in data:
 355             try:
 356                 ep = int(data['episodeNumber'])
 357             except:
 358                 ep = 0
 359             data['episodeNumber'] = str(ep)
 360
 361         if config.getDebug() and 'vHost' not in data:
 362             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 363             if compatible:
 364                 transcode_options = []
 365             else:
 366                 transcode_options = transcode.transcode(True, full_path,
 367                                                         '', tsn, mime)
 368             data['vHost'] = (
 369                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 370                 ['SOURCE INFO: '] +
 371                 ["%s=%s" % (k, v)
 372                  for k, v in sorted(vInfo.items(), reverse=True)] +
 373                 ['TRANSCODE OPTIONS: '] +
 374                 transcode_options +
 375                 ['SOURCE FILE: ', os.path.basename(full_path)]
 376             )
 377
 378         now = datetime.utcnow()
 379         if 'time' in data:
 380             if data['time'].lower() == 'file':
 381                 if not mtime:
 382                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 383                 try:
 384                     now = datetime.utcfromtimestamp(mtime)
 385                 except:
 386                     logger.warning('Bad file time on ' + full_path)
 387             elif data['time'].lower() == 'oad':
 388                     now = isodt(data['originalAirDate'])
 389             else:
 390                 try:
 391                     now = isodt(data['time'])
 392                 except:
 393                     logger.warning('Bad time format: ' + data['time'] +
 394                                    ' , using current time')
 395
 396         duration = self.__duration(full_path)
 397         duration_delta = timedelta(milliseconds = duration)
 398         min = duration_delta.seconds / 60
 399         sec = duration_delta.seconds % 60
 400         hours = min / 60
 401         min = min % 60
 402
 403         data.update({'time': now.isoformat(),
 404                      'startTime': now.isoformat(),
 405                      'stopTime': (now + duration_delta).isoformat(),
 406                      'size': self.__est_size(full_path, tsn, mime),
 407                      'duration': duration,
 408                      'iso_duration': ('P%sDT%sH%sM%sS' %
 409                           (duration_delta.days, hours, min, sec))})
 410
 411         return data
 412
 413     def QueryContainer(self, handler, query):
 414         tsn = handler.headers.getheader('tsn', '')
 415         subcname = query['Container'][0]
 416
 417         if not self.get_local_path(handler, query):
 418             handler.send_error(404)
 419             return
 420
 421         container = handler.container
 422         force_alpha = container.getboolean('force_alpha')
 423         ar = container.get('allow_recurse', 'auto').lower()
 424         if ar == 'auto':
 425             allow_recurse = not tsn or tsn[0] < '7'
 426         else:
 427             allow_recurse = ar in ('1', 'yes', 'true', 'on')
 428         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 429
 430         files, total, start = self.get_files(handler, query,
 431                                              self.video_file_filter,
 432                                              force_alpha, allow_recurse)
 433
 434         videos = []
 435         local_base_path = self.get_local_base_path(handler, query)
 436         for f in files:
 437             video = VideoDetails()
 438             mtime = f.mdate
 439             try:
 440                 ltime = time.localtime(mtime)
 441             except:
 442                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 443                 mtime = time.time()
 444                 ltime = time.localtime(mtime)
 445             video['captureDate'] = hex(int(mtime))
 446             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 447             video['name'] = os.path.basename(f.name)
 448             video['path'] = f.name
 449             video['part_path'] = f.name.replace(local_base_path, '', 1)
 450             if not video['part_path'].startswith(os.path.sep):
 451                 video['part_path'] = os.path.sep + video['part_path']
 452             video['title'] = os.path.basename(f.name)
 453             video['is_dir'] = f.isdir
 454             if video['is_dir']:
 455                 video['small_path'] = subcname + '/' + video['name']
 456                 video['total_items'] = self.__total_items(f.name)
 457             else:
 458                 if len(files) == 1 or f.name in transcode.info_cache:
 459                     video['valid'] = transcode.supported_format(f.name)
 460                     if video['valid']:
 461                         video.update(self.metadata_full(f.name, tsn,
 462                                      mtime=mtime))
 463                         if len(files) == 1:
 464                             video['captureDate'] = hex(isogm(video['time']))
 465                 else:
 466                     video['valid'] = True
 467                     video.update(metadata.basic(f.name, mtime))
 468
 469                 if self.use_ts(tsn, f.name):
 470                     video['mime'] = 'video/x-tivo-mpeg-ts'
 471                 else:
 472                     video['mime'] = 'video/x-tivo-mpeg'
 473
 474                 video['textSize'] = metadata.human_size(f.size)
 475
 476             videos.append(video)
 477
 478         if use_html:
 479             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 480         else:
 481             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 482         t.container = handler.cname
 483         t.name = subcname
 484         t.total = total
 485         t.start = start
 486         t.videos = videos
 487         t.quote = quote
 488         t.escape = escape
 489         t.crc = zlib.crc32
 490         t.guid = config.getGUID()
 491         t.tivos = config.tivos
 492         if use_html:
 493             handler.send_html(str(t))
 494         else:
 495             handler.send_xml(str(t))
 496
 497     def use_ts(self, tsn, file_path):
 498         if config.is_ts_capable(tsn):
 499             ext = os.path.splitext(file_path)[1].lower()
 500             if ext == '.tivo':
 501                 try:
 502                     flag = file(file_path).read(8)
 503                 except:
 504                     return False
 505                 if ord(flag[7]) & 0x20:
 506                     return True
 507             else:
 508                 opt = config.get_ts_flag()
 509                 if ((opt == 'auto' and ext in LIKELYTS) or
 510                     (opt in ['true', 'yes', 'on'])):
 511                     return True
 512
 513         return False
 514
 515     def get_details_xml(self, tsn, file_path):
 516         if (tsn, file_path) in self.tvbus_cache:
 517             details = self.tvbus_cache[(tsn, file_path)]
 518         else:
 519             file_info = VideoDetails()
 520             file_info['valid'] = transcode.supported_format(file_path)
 521             if file_info['valid']:
 522                 file_info.update(self.metadata_full(file_path, tsn))
 523
 524             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 525             t.video = file_info
 526             t.escape = escape
 527             t.get_tv = metadata.get_tv
 528             t.get_mpaa = metadata.get_mpaa
 529             t.get_stars = metadata.get_stars
 530             t.get_color = metadata.get_color
 531             details = str(t)
 532             self.tvbus_cache[(tsn, file_path)] = details
 533         return details
 534
 535     def tivo_header(self, tsn, path, mime):
 536         def pad(length, align):
 537             extra = length % align
 538             if extra:
 539                 extra = align - extra
 540             return extra
 541
 542         if mime == 'video/x-tivo-mpeg-ts':
 543             flag = 45
 544         else:
 545             flag = 13
 546         details = self.get_details_xml(tsn, path)
 547         ld = len(details)
 548         chunk = details + '\0' * (pad(ld, 4) + 4)
 549         lc = len(chunk)
 550         blocklen = lc * 2 + 40
 551         padding = pad(blocklen, 1024)
 552
 553         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 554                                             padding + blocklen, 2),
 555                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 556                         chunk,
 557                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 558                         chunk, '\0' * padding])
 559
 560     def TVBusQuery(self, handler, query):
 561         tsn = handler.headers.getheader('tsn', '')
 562         f = query['File'][0]
 563         path = self.get_local_path(handler, query)
 564         file_path = os.path.normpath(path + '/' + f)
 565
 566         details = self.get_details_xml(tsn, file_path)
 567
 568         handler.send_xml(details)
 569
 570 class Video(BaseVideo, Pushable):
 571         pass
 572
 573 class VideoDetails(DictMixin):
 574
 575     def __init__(self, d=None):
 576         if d:
 577             self.d = d
 578         else:
 579             self.d = {}
 580
 581     def __getitem__(self, key):
 582         if key not in self.d:
 583             self.d[key] = self.default(key)
 584         return self.d[key]
 585
 586     def __contains__(self, key):
 587         return True
 588
 589     def __setitem__(self, key, value):
 590         self.d[key] = value
 591
 592     def __delitem__(self):
 593         del self.d[key]
 594
 595     def keys(self):
 596         return self.d.keys()
 597
 598     def __iter__(self):
 599         return self.d.__iter__()
 600
 601     def iteritems(self):
 602         return self.d.iteritems()
 603
 604     def default(self, key):
 605         defaults = {
 606             'showingBits' : '0',
 607             'displayMajorNumber' : '0',
 608             'displayMinorNumber' : '0',
 609             'isEpisode' : 'true',
 610             'colorCode' : '4',
 611             'showType' : ('SERIES', '5')
 612         }
 613         if key in defaults:
 614             return defaults[key]
 615         elif key.startswith('v'):
 616             return []
 617         else:
 618             return ''