plugins/video/video.py

   1 import calendar
   2 import cgi
   3 import logging
   4 import os
   5 import re
   6 import struct
   7 import thread
   8 import time
   9 import traceback
  10 import urllib
  11 import zlib
  12 from UserDict import DictMixin
  13 from datetime import datetime, timedelta
  14 from xml.sax.saxutils import escape
  15
  16 from Cheetah.Template import Template
  17 from lrucache import LRUCache
  18
  19 import config
  20 import metadata
  21 import mind
  22 import qtfaststart
  23 import transcode
  24 from plugin import EncodeUnicode, Plugin, quote
  25
  26 logger = logging.getLogger('pyTivo.video.video')
  27
  28 SCRIPTDIR = os.path.dirname(__file__)
  29
  30 CLASS_NAME = 'Video'
  31
  32 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  33
  34 # Preload the templates
  35 def tmpl(name):
  36     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  37
  38 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  39 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  40 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  41
  42 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  43 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  44 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  45 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  46 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  47 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  48 .wm .wmd .wtv .yuv""".split()
  49
  50 use_extensions = True
  51 try:
  52     assert(config.get_bin('ffmpeg'))
  53 except:
  54     use_extensions = False
  55
  56 queue = []  # Recordings to push
  57
  58 def uniso(iso):
  59     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  60
  61 def isodt(iso):
  62     return datetime(*uniso(iso)[:6])
  63
  64 def isogm(iso):
  65     return int(calendar.timegm(uniso(iso)))
  66
  67 class Pushable(object):
  68
  69     def push_one_file(self, f):
  70         file_info = VideoDetails()
  71         file_info['valid'] = transcode.supported_format(f['path'])
  72
  73         mime = 'video/mpeg'
  74         if config.isHDtivo(f['tsn']):
  75             for m in ['video/mp4', 'video/bif']:
  76                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  77                     mime = m
  78                     break
  79
  80             if (mime == 'video/mpeg' and
  81                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  82                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  83                 if new_path:
  84                     mime = 'video/mp4'
  85                     f['name'] = new_path
  86
  87         if file_info['valid']:
  88             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  89
  90         url = f['url'] + quote(f['name'])
  91
  92         title = file_info['seriesTitle']
  93         if not title:
  94             title = file_info['title']
  95
  96         source = file_info['seriesId']
  97         if not source:
  98             source = title
  99
 100         subtitle = file_info['episodeTitle']
 101         try:
 102             m = mind.getMind(f['tsn'])
 103             m.pushVideo(
 104                 tsn = f['tsn'],
 105                 url = url,
 106                 description = file_info['description'],
 107                 duration = file_info['duration'] / 1000,
 108                 size = file_info['size'],
 109                 title = title,
 110                 subtitle = subtitle,
 111                 source = source,
 112                 mime = mime,
 113                 tvrating = file_info['tvRating'])
 114         except Exception, msg:
 115             logger.error(msg)
 116
 117     def process_queue(self):
 118         while queue:
 119             time.sleep(5)
 120             item = queue.pop(0)
 121             self.push_one_file(item)
 122
 123     def readip(self):
 124         """ returns your external IP address by querying dyndns.org """
 125         f = urllib.urlopen('http://checkip.dyndns.org/')
 126         s = f.read()
 127         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 128         return m.group(0)
 129
 130     def Push(self, handler, query):
 131         tsn = query['tsn'][0]
 132         for key in config.tivo_names:
 133             if config.tivo_names[key] == tsn:
 134                 tsn = key
 135                 break
 136         tivo_name = config.tivo_names.get(tsn, tsn)
 137
 138         container = quote(query['Container'][0].split('/')[0])
 139         ip = config.get_ip(tsn)
 140         port = config.getPort()
 141
 142         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 143         if config.getIsExternal(tsn):
 144             exturl = config.get_server('externalurl')
 145             if exturl:
 146                 if not exturl.endswith('/'):
 147                     exturl += '/'
 148                 baseurl = exturl + container
 149             else:
 150                 ip = self.readip()
 151                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 152
 153         path = self.get_local_base_path(handler, query)
 154
 155         files = query.get('File', [])
 156         for f in files:
 157             file_path = path + os.path.normpath(f)
 158             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 159                           'url': baseurl})
 160             if len(queue) == 1:
 161                 thread.start_new_thread(Video.process_queue, (self,))
 162
 163             logger.info('[%s] Queued "%s" for Push to %s' %
 164                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 165                          unicode(file_path, 'utf-8'), tivo_name))
 166
 167         files = [unicode(f, 'utf-8') for f in files]
 168         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 169
 170 class BaseVideo(Plugin):
 171
 172     CONTENT_TYPE = 'x-container/tivo-videos'
 173
 174     tvbus_cache = LRUCache(1)
 175
 176     def pre_cache(self, full_path):
 177         if Video.video_file_filter(self, full_path):
 178             transcode.supported_format(full_path)
 179
 180     def video_file_filter(self, full_path, type=None):
 181         if os.path.isdir(unicode(full_path, 'utf-8')):
 182             return True
 183         if use_extensions:
 184             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 185         else:
 186             return transcode.supported_format(full_path)
 187
 188     def send_file(self, handler, path, query):
 189         mime = 'video/x-tivo-mpeg'
 190         tsn = handler.headers.getheader('tsn', '')
 191         tivo_name = config.tivo_names.get(tsn, tsn)
 192
 193         is_tivo_file = (path[-5:].lower() == '.tivo')
 194
 195         if 'Format' in query:
 196             mime = query['Format'][0]
 197
 198         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 199         compatible = (not needs_tivodecode and
 200                       transcode.tivo_compatible(path, tsn, mime)[0])
 201
 202         try:  # "bytes=XXX-"
 203             offset = int(handler.headers.getheader('Range')[6:-1])
 204         except:
 205             offset = 0
 206
 207         if needs_tivodecode:
 208             valid = bool(config.get_bin('tivodecode') and
 209                          config.get_server('tivo_mak'))
 210         else:
 211             valid = True
 212
 213         if valid and offset:
 214             valid = ((compatible and offset < os.stat(path).st_size) or
 215                      (not compatible and transcode.is_resumable(path, offset)))
 216
 217         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 218         faking = (mime == 'video/x-tivo-mpeg' and
 219                   not (is_tivo_file and compatible))
 220         fname = unicode(path, 'utf-8')
 221         thead = ''
 222         if faking:
 223             thead = self.tivo_header(tsn, path, mime)
 224         if compatible:
 225             size = os.stat(fname).st_size + len(thead)
 226             handler.send_response(200)
 227             handler.send_header('Content-Length', size - offset)
 228             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 229                                 (offset, size - offset - 1, size))
 230         else:
 231             handler.send_response(206)
 232             handler.send_header('Transfer-Encoding', 'chunked')
 233         handler.send_header('Content-Type', mime)
 234         handler.send_header('Connection', 'close')
 235         handler.end_headers()
 236
 237         logger.info('[%s] Start sending "%s" to %s' %
 238                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 239         start = time.time()
 240         count = 0
 241
 242         if valid:
 243             if compatible:
 244                 if faking and not offset:
 245                     handler.wfile.write(thead)
 246                 logger.debug('"%s" is tivo compatible' % fname)
 247                 f = open(fname, 'rb')
 248                 try:
 249                     if mime == 'video/mp4':
 250                         count = qtfaststart.process(f, handler.wfile, offset)
 251                     else:
 252                         if offset:
 253                             offset -= len(thead)
 254                             f.seek(offset)
 255                         while True:
 256                             block = f.read(512 * 1024)
 257                             if not block:
 258                                 break
 259                             handler.wfile.write(block)
 260                             count += len(block)
 261                 except Exception, msg:
 262                     logger.info(msg)
 263                 f.close()
 264             else:
 265                 logger.debug('"%s" is not tivo compatible' % fname)
 266                 if offset:
 267                     count = transcode.resume_transfer(path, handler.wfile,
 268                                                       offset)
 269                 else:
 270                     count = transcode.transcode(False, path, handler.wfile,
 271                                                 tsn, mime, thead)
 272         try:
 273             if not compatible:
 274                  handler.wfile.write('0\r\n\r\n')
 275             handler.wfile.flush()
 276         except Exception, msg:
 277             logger.info(msg)
 278
 279         mega_elapsed = (time.time() - start) * 1024 * 1024
 280         if mega_elapsed < 1:
 281             mega_elapsed = 1
 282         rate = count * 8.0 / mega_elapsed
 283         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 284                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 285                      tivo_name, count, rate))
 286
 287         if fname.endswith('.pyTivo-temp'):
 288             os.remove(fname)
 289
 290     def __duration(self, full_path):
 291         return transcode.video_info(full_path)['millisecs']
 292
 293     def __total_items(self, full_path):
 294         count = 0
 295         try:
 296             full_path = unicode(full_path, 'utf-8')
 297             for f in os.listdir(full_path):
 298                 if f.startswith('.'):
 299                     continue
 300                 f = os.path.join(full_path, f)
 301                 f2 = f.encode('utf-8')
 302                 if os.path.isdir(f):
 303                     count += 1
 304                 elif use_extensions:
 305                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 306                         count += 1
 307                 elif f2 in transcode.info_cache:
 308                     if transcode.supported_format(f2):
 309                         count += 1
 310         except:
 311             pass
 312         return count
 313
 314     def __est_size(self, full_path, tsn='', mime=''):
 315         # Size is estimated by taking audio and video bit rate adding 2%
 316
 317         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 318             return int(os.stat(unicode(full_path, 'utf-8')).st_size)
 319         else:
 320             # Must be re-encoded
 321             if config.get_tsn('audio_codec', tsn) == None:
 322                 audioBPS = config.getMaxAudioBR(tsn) * 1000
 323             else:
 324                 audioBPS = config.strtod(config.getAudioBR(tsn))
 325             videoBPS = transcode.select_videostr(full_path, tsn)
 326             bitrate =  audioBPS + videoBPS
 327             return int((self.__duration(full_path) / 1000) *
 328                        (bitrate * 1.02 / 8))
 329
 330     def metadata_full(self, full_path, tsn='', mime=''):
 331         data = {}
 332         vInfo = transcode.video_info(full_path)
 333
 334         if ((int(vInfo['vHeight']) >= 720 and
 335              config.getTivoHeight >= 720) or
 336             (int(vInfo['vWidth']) >= 1280 and
 337              config.getTivoWidth >= 1280)):
 338             data['showingBits'] = '4096'
 339
 340         data.update(metadata.basic(full_path))
 341         if full_path[-5:].lower() == '.tivo':
 342             data.update(metadata.from_tivo(full_path))
 343         if full_path[-4:].lower() == '.wtv':
 344             data.update(metadata.from_mscore(vInfo['rawmeta']))
 345
 346         if 'episodeNumber' in data:
 347             try:
 348                 ep = int(data['episodeNumber'])
 349             except:
 350                 ep = 0
 351             data['episodeNumber'] = str(ep)
 352
 353         if config.getDebug() and 'vHost' not in data:
 354             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 355             if compatible:
 356                 transcode_options = {}
 357             else:
 358                 transcode_options = transcode.transcode(True, full_path,
 359                                                         '', tsn, mime)
 360             data['vHost'] = (
 361                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 362                 ['SOURCE INFO: '] +
 363                 ["%s=%s" % (k, v)
 364                  for k, v in sorted(vInfo.items(), reverse=True)] +
 365                 ['TRANSCODE OPTIONS: '] +
 366                 ["%s" % (v) for k, v in transcode_options.items()] +
 367                 ['SOURCE FILE: ', os.path.basename(full_path)]
 368             )
 369
 370         now = datetime.utcnow()
 371         if 'time' in data:
 372             if data['time'].lower() == 'file':
 373                 mtime = os.stat(unicode(full_path, 'utf-8')).st_mtime
 374                 if (mtime < 0):
 375                     mtime = 0
 376                 try:
 377                     now = datetime.utcfromtimestamp(mtime)
 378                 except:
 379                     logger.warning('Bad file time on ' + full_path)
 380             elif data['time'].lower() == 'oad':
 381                     now = isodt(data['originalAirDate'])
 382             else:
 383                 try:
 384                     now = isodt(data['time'])
 385                 except:
 386                     logger.warning('Bad time format: ' + data['time'] +
 387                                    ' , using current time')
 388
 389         duration = self.__duration(full_path)
 390         duration_delta = timedelta(milliseconds = duration)
 391         min = duration_delta.seconds / 60
 392         sec = duration_delta.seconds % 60
 393         hours = min / 60
 394         min = min % 60
 395
 396         data.update({'time': now.isoformat(),
 397                      'startTime': now.isoformat(),
 398                      'stopTime': (now + duration_delta).isoformat(),
 399                      'size': self.__est_size(full_path, tsn, mime),
 400                      'duration': duration,
 401                      'iso_duration': ('P%sDT%sH%sM%sS' %
 402                           (duration_delta.days, hours, min, sec))})
 403
 404         return data
 405
 406     def QueryContainer(self, handler, query):
 407         tsn = handler.headers.getheader('tsn', '')
 408         subcname = query['Container'][0]
 409
 410         if not self.get_local_path(handler, query):
 411             handler.send_error(404)
 412             return
 413
 414         container = handler.container
 415         force_alpha = container.get('force_alpha', 'False').lower() == 'true'
 416         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 417
 418         files, total, start = self.get_files(handler, query,
 419                                              self.video_file_filter,
 420                                              force_alpha)
 421
 422         videos = []
 423         local_base_path = self.get_local_base_path(handler, query)
 424         for f in files:
 425             video = VideoDetails()
 426             mtime = f.mdate
 427             try:
 428                 ltime = time.localtime(mtime)
 429             except:
 430                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 431                 mtime = int(time.time())
 432                 ltime = time.localtime(mtime)
 433             video['captureDate'] = hex(mtime)
 434             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 435             video['name'] = os.path.basename(f.name)
 436             video['path'] = f.name
 437             video['part_path'] = f.name.replace(local_base_path, '', 1)
 438             if not video['part_path'].startswith(os.path.sep):
 439                 video['part_path'] = os.path.sep + video['part_path']
 440             video['title'] = os.path.basename(f.name)
 441             video['is_dir'] = f.isdir
 442             if video['is_dir']:
 443                 video['small_path'] = subcname + '/' + video['name']
 444                 video['total_items'] = self.__total_items(f.name)
 445             else:
 446                 if len(files) == 1 or f.name in transcode.info_cache:
 447                     video['valid'] = transcode.supported_format(f.name)
 448                     if video['valid']:
 449                         video.update(self.metadata_full(f.name, tsn))
 450                         if len(files) == 1:
 451                             video['captureDate'] = hex(isogm(video['time']))
 452                 else:
 453                     video['valid'] = True
 454                     video.update(metadata.basic(f.name))
 455
 456                 if self.use_ts(tsn, f.name):
 457                     video['mime'] = 'video/x-tivo-mpeg-ts'
 458                 else:
 459                     video['mime'] = 'video/x-tivo-mpeg'
 460
 461                 video['textSize'] = ( '%.3f GB' %
 462                     (float(f.size) / (1024 ** 3)) )
 463
 464             videos.append(video)
 465
 466         if use_html:
 467             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 468         else:
 469             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 470         t.container = handler.cname
 471         t.name = subcname
 472         t.total = total
 473         t.start = start
 474         t.videos = videos
 475         t.quote = quote
 476         t.escape = escape
 477         t.crc = zlib.crc32
 478         t.guid = config.getGUID()
 479         t.tivos = config.tivos
 480         t.tivo_names = config.tivo_names
 481         if use_html:
 482             handler.send_html(str(t))
 483         else:
 484             handler.send_xml(str(t))
 485
 486     def use_ts(self, tsn, file_path):
 487         if config.is_ts_capable(tsn):
 488             if file_path[-5:].lower() == '.tivo':
 489                 try:
 490                     flag = file(file_path).read(8)
 491                 except:
 492                     return False
 493                 if ord(flag[7]) & 0x20:
 494                     return True
 495             elif config.has_ts_flag():
 496                 return True
 497
 498         return False
 499
 500     def get_details_xml(self, tsn, file_path):
 501         if (tsn, file_path) in self.tvbus_cache:
 502             details = self.tvbus_cache[(tsn, file_path)]
 503         else:
 504             file_info = VideoDetails()
 505             file_info['valid'] = transcode.supported_format(file_path)
 506             if file_info['valid']:
 507                 file_info.update(self.metadata_full(file_path, tsn))
 508
 509             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 510             t.video = file_info
 511             t.escape = escape
 512             t.get_tv = metadata.get_tv
 513             t.get_mpaa = metadata.get_mpaa
 514             t.get_stars = metadata.get_stars
 515             details = str(t)
 516             self.tvbus_cache[(tsn, file_path)] = details
 517         return details
 518
 519     def tivo_header(self, tsn, path, mime):
 520         if mime == 'video/x-tivo-mpeg-ts':
 521             flag = 45
 522         else:
 523             flag = 13
 524         details = self.get_details_xml(tsn, path)
 525         ld = len(details)
 526         chunklen = ld * 2 + 44
 527         padding = 2048 - chunklen % 1024
 528
 529         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 530                                             padding + chunklen, 2),
 531                         struct.pack('>LLHH', ld + 16, ld, 1, 0),
 532                         details, '\0' * 4,
 533                         struct.pack('>LLHH', ld + 19, ld, 2, 0),
 534                         details, '\0' * padding])
 535
 536     def TVBusQuery(self, handler, query):
 537         tsn = handler.headers.getheader('tsn', '')
 538         f = query['File'][0]
 539         path = self.get_local_path(handler, query)
 540         file_path = path + os.path.normpath(f)
 541
 542         details = self.get_details_xml(tsn, file_path)
 543
 544         handler.send_xml(details)
 545
 546 class Video(BaseVideo, Pushable):
 547         pass
 548
 549 class VideoDetails(DictMixin):
 550
 551     def __init__(self, d=None):
 552         if d:
 553             self.d = d
 554         else:
 555             self.d = {}
 556
 557     def __getitem__(self, key):
 558         if key not in self.d:
 559             self.d[key] = self.default(key)
 560         return self.d[key]
 561
 562     def __contains__(self, key):
 563         return True
 564
 565     def __setitem__(self, key, value):
 566         self.d[key] = value
 567
 568     def __delitem__(self):
 569         del self.d[key]
 570
 571     def keys(self):
 572         return self.d.keys()
 573
 574     def __iter__(self):
 575         return self.d.__iter__()
 576
 577     def iteritems(self):
 578         return self.d.iteritems()
 579
 580     def default(self, key):
 581         defaults = {
 582             'showingBits' : '0',
 583             'displayMajorNumber' : '0',
 584             'displayMinorNumber' : '0',
 585             'isEpisode' : 'true',
 586             'colorCode' : ('COLOR', '4'),
 587             'showType' : ('SERIES', '5')
 588         }
 589         if key in defaults:
 590             return defaults[key]
 591         elif key.startswith('v'):
 592             return []
 593         else:
 594             return ''