plugins/video/video.py

   1 import calendar
   2 import logging
   3 import os
   4 import re
   5 import struct
   6 import thread
   7 import time
   8 import urllib
   9 import zlib
  10 from UserDict import DictMixin
  11 from datetime import datetime, timedelta
  12 from xml.sax.saxutils import escape
  13
  14 from Cheetah.Template import Template
  15 from lrucache import LRUCache
  16
  17 import config
  18 import metadata
  19 import mind
  20 import qtfaststart
  21 import transcode
  22 from plugin import EncodeUnicode, Plugin, quote
  23
  24 logger = logging.getLogger('pyTivo.video.video')
  25
  26 SCRIPTDIR = os.path.dirname(__file__)
  27
  28 CLASS_NAME = 'Video'
  29
  30 PUSHED = '<h3>Queued for Push to %s</h3> <p>%s</p>'
  31
  32 # Preload the templates
  33 def tmpl(name):
  34     return file(os.path.join(SCRIPTDIR, 'templates', name), 'rb').read()
  35
  36 HTML_CONTAINER_TEMPLATE = tmpl('container_html.tmpl')
  37 XML_CONTAINER_TEMPLATE = tmpl('container_xml.tmpl')
  38 TVBUS_TEMPLATE = tmpl('TvBus.tmpl')
  39
  40 EXTENSIONS = """.tivo .mpg .avi .wmv .mov .flv .f4v .vob .mp4 .m4v .mkv
  41 .ts .tp .trp .3g2 .3gp .3gp2 .3gpp .amv .asf .avs .bik .bix .box .bsf
  42 .dat .dif .divx .dmb .dpg .dv .dvr-ms .evo .eye .flc .fli .flx .gvi .ivf
  43 .m1v .m21 .m2t .m2ts .m2v .m2p .m4e .mjp .mjpeg .mod .moov .movie .mp21
  44 .mpe .mpeg .mpv .mpv2 .mqv .mts .mvb .nsv .nuv .nut .ogm .qt .rm .rmvb
  45 .rts .scm .smv .ssm .svi .vdo .vfw .vid .viv .vivo .vp6 .vp7 .vro .webm
  46 .wm .wmd .wtv .yuv""".split()
  47
  48 use_extensions = True
  49 try:
  50     assert(config.get_bin('ffmpeg'))
  51 except:
  52     use_extensions = False
  53
  54 queue = []  # Recordings to push
  55
  56 def uniso(iso):
  57     return time.strptime(iso[:19], '%Y-%m-%dT%H:%M:%S')
  58
  59 def isodt(iso):
  60     return datetime(*uniso(iso)[:6])
  61
  62 def isogm(iso):
  63     return int(calendar.timegm(uniso(iso)))
  64
  65 class Pushable(object):
  66
  67     def push_one_file(self, f):
  68         file_info = VideoDetails()
  69         file_info['valid'] = transcode.supported_format(f['path'])
  70
  71         mime = 'video/mpeg'
  72         if config.isHDtivo(f['tsn']):
  73             for m in ['video/mp4', 'video/bif']:
  74                 if transcode.tivo_compatible(f['path'], f['tsn'], m)[0]:
  75                     mime = m
  76                     break
  77
  78             if (mime == 'video/mpeg' and
  79                 transcode.mp4_remuxable(f['path'], f['tsn'])):
  80                 new_path = transcode.mp4_remux(f['path'], f['name'], f['tsn'])
  81                 if new_path:
  82                     mime = 'video/mp4'
  83                     f['name'] = new_path
  84
  85         if file_info['valid']:
  86             file_info.update(self.metadata_full(f['path'], f['tsn'], mime))
  87
  88         url = f['url'] + quote(f['name'])
  89
  90         title = file_info['seriesTitle']
  91         if not title:
  92             title = file_info['title']
  93
  94         source = file_info['seriesId']
  95         if not source:
  96             source = title
  97
  98         subtitle = file_info['episodeTitle']
  99         try:
 100             m = mind.getMind(f['tsn'])
 101             m.pushVideo(
 102                 tsn = f['tsn'],
 103                 url = url,
 104                 description = file_info['description'],
 105                 duration = file_info['duration'] / 1000,
 106                 size = file_info['size'],
 107                 title = title,
 108                 subtitle = subtitle,
 109                 source = source,
 110                 mime = mime,
 111                 tvrating = file_info['tvRating'])
 112         except Exception, msg:
 113             logger.error(msg)
 114
 115     def process_queue(self):
 116         while queue:
 117             time.sleep(5)
 118             item = queue.pop(0)
 119             self.push_one_file(item)
 120
 121     def readip(self):
 122         """ returns your external IP address by querying dyndns.org """
 123         f = urllib.urlopen('http://checkip.dyndns.org/')
 124         s = f.read()
 125         m = re.search('([\d]*\.[\d]*\.[\d]*\.[\d]*)', s)
 126         return m.group(0)
 127
 128     def Push(self, handler, query):
 129         tsn = query['tsn'][0]
 130         for key in config.tivos:
 131             if config.tivos[key]['name'] == tsn:
 132                 tsn = key
 133                 break
 134         tivo_name = config.tivos[tsn].get('name', tsn)
 135
 136         container = quote(query['Container'][0].split('/')[0])
 137         ip = config.get_ip(tsn)
 138         port = config.getPort()
 139
 140         baseurl = 'http://%s:%s/%s' % (ip, port, container)
 141         if config.getIsExternal(tsn):
 142             exturl = config.get_server('externalurl')
 143             if exturl:
 144                 if not exturl.endswith('/'):
 145                     exturl += '/'
 146                 baseurl = exturl + container
 147             else:
 148                 ip = self.readip()
 149                 baseurl = 'http://%s:%s/%s' % (ip, port, container)
 150
 151         path = self.get_local_base_path(handler, query)
 152
 153         files = query.get('File', [])
 154         for f in files:
 155             file_path = os.path.normpath(path + '/' + f)
 156             queue.append({'path': file_path, 'name': f, 'tsn': tsn,
 157                           'url': baseurl})
 158             if len(queue) == 1:
 159                 thread.start_new_thread(Video.process_queue, (self,))
 160
 161             logger.info('[%s] Queued "%s" for Push to %s' %
 162                         (time.strftime('%d/%b/%Y %H:%M:%S'),
 163                          unicode(file_path, 'utf-8'), tivo_name))
 164
 165         files = [unicode(f, 'utf-8') for f in files]
 166         handler.redir(PUSHED % (tivo_name, '<br>'.join(files)), 5)
 167
 168 class BaseVideo(Plugin):
 169
 170     CONTENT_TYPE = 'x-container/tivo-videos'
 171
 172     tvbus_cache = LRUCache(1)
 173
 174     def video_file_filter(self, full_path, type=None):
 175         if os.path.isdir(unicode(full_path, 'utf-8')):
 176             return True
 177         if use_extensions:
 178             return os.path.splitext(full_path)[1].lower() in EXTENSIONS
 179         else:
 180             return transcode.supported_format(full_path)
 181
 182     def send_file(self, handler, path, query):
 183         mime = 'video/x-tivo-mpeg'
 184         tsn = handler.headers.getheader('tsn', '')
 185         tivo_name = config.tivos[tsn].get('name', tsn)
 186
 187         is_tivo_file = (path[-5:].lower() == '.tivo')
 188
 189         if 'Format' in query:
 190             mime = query['Format'][0]
 191
 192         needs_tivodecode = (is_tivo_file and mime == 'video/mpeg')
 193         compatible = (not needs_tivodecode and
 194                       transcode.tivo_compatible(path, tsn, mime)[0])
 195
 196         try:  # "bytes=XXX-"
 197             offset = int(handler.headers.getheader('Range')[6:-1])
 198         except:
 199             offset = 0
 200
 201         if needs_tivodecode:
 202             valid = bool(config.get_bin('tivodecode') and
 203                          config.get_server('tivo_mak'))
 204         else:
 205             valid = True
 206
 207         if valid and offset:
 208             valid = ((compatible and offset < os.path.getsize(path)) or
 209                      (not compatible and transcode.is_resumable(path, offset)))
 210
 211         #faking = (mime in ['video/x-tivo-mpeg-ts', 'video/x-tivo-mpeg'] and
 212         faking = (mime == 'video/x-tivo-mpeg' and
 213                   not (is_tivo_file and compatible))
 214         fname = unicode(path, 'utf-8')
 215         thead = ''
 216         if faking:
 217             thead = self.tivo_header(tsn, path, mime)
 218         if compatible:
 219             size = os.path.getsize(fname) + len(thead)
 220             handler.send_response(200)
 221             handler.send_header('Content-Length', size - offset)
 222             handler.send_header('Content-Range', 'bytes %d-%d/%d' %
 223                                 (offset, size - offset - 1, size))
 224         else:
 225             handler.send_response(206)
 226             handler.send_header('Transfer-Encoding', 'chunked')
 227         handler.send_header('Content-Type', mime)
 228         handler.end_headers()
 229
 230         logger.info('[%s] Start sending "%s" to %s' %
 231                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname, tivo_name))
 232         start = time.time()
 233         count = 0
 234
 235         if valid:
 236             if compatible:
 237                 if faking and not offset:
 238                     handler.wfile.write(thead)
 239                 logger.debug('"%s" is tivo compatible' % fname)
 240                 f = open(fname, 'rb')
 241                 try:
 242                     if mime == 'video/mp4':
 243                         count = qtfaststart.process(f, handler.wfile, offset)
 244                     else:
 245                         if offset:
 246                             offset -= len(thead)
 247                             f.seek(offset)
 248                         while True:
 249                             block = f.read(512 * 1024)
 250                             if not block:
 251                                 break
 252                             handler.wfile.write(block)
 253                             count += len(block)
 254                 except Exception, msg:
 255                     logger.info(msg)
 256                 f.close()
 257             else:
 258                 logger.debug('"%s" is not tivo compatible' % fname)
 259                 if offset:
 260                     count = transcode.resume_transfer(path, handler.wfile,
 261                                                       offset)
 262                 else:
 263                     count = transcode.transcode(False, path, handler.wfile,
 264                                                 tsn, mime, thead)
 265         try:
 266             if not compatible:
 267                  handler.wfile.write('0\r\n\r\n')
 268             handler.wfile.flush()
 269         except Exception, msg:
 270             logger.info(msg)
 271
 272         mega_elapsed = (time.time() - start) * 1024 * 1024
 273         if mega_elapsed < 1:
 274             mega_elapsed = 1
 275         rate = count * 8.0 / mega_elapsed
 276         logger.info('[%s] Done sending "%s" to %s, %d bytes, %.2f Mb/s' %
 277                     (time.strftime('%d/%b/%Y %H:%M:%S'), fname,
 278                      tivo_name, count, rate))
 279
 280         if fname.endswith('.pyTivo-temp'):
 281             os.remove(fname)
 282
 283     def __duration(self, full_path):
 284         return transcode.video_info(full_path)['millisecs']
 285
 286     def __total_items(self, full_path):
 287         count = 0
 288         try:
 289             full_path = unicode(full_path, 'utf-8')
 290             for f in os.listdir(full_path):
 291                 if f.startswith('.'):
 292                     continue
 293                 f = os.path.join(full_path, f)
 294                 f2 = f.encode('utf-8')
 295                 if os.path.isdir(f):
 296                     count += 1
 297                 elif use_extensions:
 298                     if os.path.splitext(f2)[1].lower() in EXTENSIONS:
 299                         count += 1
 300                 elif f2 in transcode.info_cache:
 301                     if transcode.supported_format(f2):
 302                         count += 1
 303         except:
 304             pass
 305         return count
 306
 307     def __est_size(self, full_path, tsn='', mime=''):
 308         # Size is estimated by taking audio and video bit rate adding 2%
 309
 310         if transcode.tivo_compatible(full_path, tsn, mime)[0]:
 311             return os.path.getsize(unicode(full_path, 'utf-8'))
 312         else:
 313             # Must be re-encoded
 314             audioBPS = config.getMaxAudioBR(tsn) * 1000
 315             #audioBPS = config.strtod(config.getAudioBR(tsn))
 316             videoBPS = transcode.select_videostr(full_path, tsn)
 317             bitrate =  audioBPS + videoBPS
 318             return int((self.__duration(full_path) / 1000) *
 319                        (bitrate * 1.02 / 8))
 320
 321     def metadata_full(self, full_path, tsn='', mime='', mtime=None):
 322         data = {}
 323         vInfo = transcode.video_info(full_path)
 324
 325         if ((int(vInfo['vHeight']) >= 720 and
 326              config.getTivoHeight >= 720) or
 327             (int(vInfo['vWidth']) >= 1280 and
 328              config.getTivoWidth >= 1280)):
 329             data['showingBits'] = '4096'
 330
 331         data.update(metadata.basic(full_path, mtime))
 332         if full_path[-5:].lower() == '.tivo':
 333             data.update(metadata.from_tivo(full_path))
 334         if full_path[-4:].lower() == '.wtv':
 335             data.update(metadata.from_mscore(vInfo['rawmeta']))
 336
 337         if 'episodeNumber' in data:
 338             try:
 339                 ep = int(data['episodeNumber'])
 340             except:
 341                 ep = 0
 342             data['episodeNumber'] = str(ep)
 343
 344         if config.getDebug() and 'vHost' not in data:
 345             compatible, reason = transcode.tivo_compatible(full_path, tsn, mime)
 346             if compatible:
 347                 transcode_options = {}
 348             else:
 349                 transcode_options = transcode.transcode(True, full_path,
 350                                                         '', tsn, mime)
 351             data['vHost'] = (
 352                 ['TRANSCODE=%s, %s' % (['YES', 'NO'][compatible], reason)] +
 353                 ['SOURCE INFO: '] +
 354                 ["%s=%s" % (k, v)
 355                  for k, v in sorted(vInfo.items(), reverse=True)] +
 356                 ['TRANSCODE OPTIONS: '] +
 357                 ["%s" % (v) for k, v in transcode_options.items()] +
 358                 ['SOURCE FILE: ', os.path.basename(full_path)]
 359             )
 360
 361         now = datetime.utcnow()
 362         if 'time' in data:
 363             if data['time'].lower() == 'file':
 364                 if not mtime:
 365                     mtime = os.path.getmtime(unicode(full_path, 'utf-8'))
 366                 try:
 367                     now = datetime.utcfromtimestamp(mtime)
 368                 except:
 369                     logger.warning('Bad file time on ' + full_path)
 370             elif data['time'].lower() == 'oad':
 371                     now = isodt(data['originalAirDate'])
 372             else:
 373                 try:
 374                     now = isodt(data['time'])
 375                 except:
 376                     logger.warning('Bad time format: ' + data['time'] +
 377                                    ' , using current time')
 378
 379         duration = self.__duration(full_path)
 380         duration_delta = timedelta(milliseconds = duration)
 381         min = duration_delta.seconds / 60
 382         sec = duration_delta.seconds % 60
 383         hours = min / 60
 384         min = min % 60
 385
 386         data.update({'time': now.isoformat(),
 387                      'startTime': now.isoformat(),
 388                      'stopTime': (now + duration_delta).isoformat(),
 389                      'size': self.__est_size(full_path, tsn, mime),
 390                      'duration': duration,
 391                      'iso_duration': ('P%sDT%sH%sM%sS' %
 392                           (duration_delta.days, hours, min, sec))})
 393
 394         return data
 395
 396     def QueryContainer(self, handler, query):
 397         tsn = handler.headers.getheader('tsn', '')
 398         subcname = query['Container'][0]
 399
 400         if not self.get_local_path(handler, query):
 401             handler.send_error(404)
 402             return
 403
 404         container = handler.container
 405         force_alpha = container.getboolean('force_alpha')
 406         use_html = query.get('Format', [''])[0].lower() == 'text/html'
 407
 408         files, total, start = self.get_files(handler, query,
 409                                              self.video_file_filter,
 410                                              force_alpha)
 411
 412         videos = []
 413         local_base_path = self.get_local_base_path(handler, query)
 414         for f in files:
 415             video = VideoDetails()
 416             mtime = f.mdate
 417             try:
 418                 ltime = time.localtime(mtime)
 419             except:
 420                 logger.warning('Bad file time on ' + unicode(f.name, 'utf-8'))
 421                 mtime = time.time()
 422                 ltime = time.localtime(mtime)
 423             video['captureDate'] = hex(int(mtime))
 424             video['textDate'] = time.strftime('%b %d, %Y', ltime)
 425             video['name'] = os.path.basename(f.name)
 426             video['path'] = f.name
 427             video['part_path'] = f.name.replace(local_base_path, '', 1)
 428             if not video['part_path'].startswith(os.path.sep):
 429                 video['part_path'] = os.path.sep + video['part_path']
 430             video['title'] = os.path.basename(f.name)
 431             video['is_dir'] = f.isdir
 432             if video['is_dir']:
 433                 video['small_path'] = subcname + '/' + video['name']
 434                 video['total_items'] = self.__total_items(f.name)
 435             else:
 436                 if len(files) == 1 or f.name in transcode.info_cache:
 437                     video['valid'] = transcode.supported_format(f.name)
 438                     if video['valid']:
 439                         video.update(self.metadata_full(f.name, tsn,
 440                                      mtime=mtime))
 441                         if len(files) == 1:
 442                             video['captureDate'] = hex(isogm(video['time']))
 443                 else:
 444                     video['valid'] = True
 445                     video.update(metadata.basic(f.name, mtime))
 446
 447                 if self.use_ts(tsn, f.name):
 448                     video['mime'] = 'video/x-tivo-mpeg-ts'
 449                 else:
 450                     video['mime'] = 'video/x-tivo-mpeg'
 451
 452                 video['textSize'] = metadata.human_size(f.size)
 453
 454             videos.append(video)
 455
 456         if use_html:
 457             t = Template(HTML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 458         else:
 459             t = Template(XML_CONTAINER_TEMPLATE, filter=EncodeUnicode)
 460         t.container = handler.cname
 461         t.name = subcname
 462         t.total = total
 463         t.start = start
 464         t.videos = videos
 465         t.quote = quote
 466         t.escape = escape
 467         t.crc = zlib.crc32
 468         t.guid = config.getGUID()
 469         t.tivos = config.tivos
 470         if use_html:
 471             handler.send_html(str(t))
 472         else:
 473             handler.send_xml(str(t))
 474
 475     def use_ts(self, tsn, file_path):
 476         if config.is_ts_capable(tsn):
 477             if file_path[-5:].lower() == '.tivo':
 478                 try:
 479                     flag = file(file_path).read(8)
 480                 except:
 481                     return False
 482                 if ord(flag[7]) & 0x20:
 483                     return True
 484             elif config.has_ts_flag():
 485                 return True
 486
 487         return False
 488
 489     def get_details_xml(self, tsn, file_path):
 490         if (tsn, file_path) in self.tvbus_cache:
 491             details = self.tvbus_cache[(tsn, file_path)]
 492         else:
 493             file_info = VideoDetails()
 494             file_info['valid'] = transcode.supported_format(file_path)
 495             if file_info['valid']:
 496                 file_info.update(self.metadata_full(file_path, tsn))
 497
 498             t = Template(TVBUS_TEMPLATE, filter=EncodeUnicode)
 499             t.video = file_info
 500             t.escape = escape
 501             t.get_tv = metadata.get_tv
 502             t.get_mpaa = metadata.get_mpaa
 503             t.get_stars = metadata.get_stars
 504             details = str(t)
 505             self.tvbus_cache[(tsn, file_path)] = details
 506         return details
 507
 508     def tivo_header(self, tsn, path, mime):
 509         def pad(length, align):
 510             extra = length % align
 511             if extra:
 512                 extra = align - extra
 513             return extra
 514
 515         if mime == 'video/x-tivo-mpeg-ts':
 516             flag = 45
 517         else:
 518             flag = 13
 519         details = self.get_details_xml(tsn, path)
 520         ld = len(details)
 521         chunk = details + '\0' * (pad(ld, 4) + 4)
 522         lc = len(chunk)
 523         blocklen = lc * 2 + 40
 524         padding = pad(blocklen, 1024)
 525
 526         return ''.join(['TiVo', struct.pack('>HHHLH', 4, flag, 0,
 527                                             padding + blocklen, 2),
 528                         struct.pack('>LLHH', lc + 12, ld, 1, 0),
 529                         chunk,
 530                         struct.pack('>LLHH', lc + 12, ld, 2, 0),
 531                         chunk, '\0' * padding])
 532
 533     def TVBusQuery(self, handler, query):
 534         tsn = handler.headers.getheader('tsn', '')
 535         f = query['File'][0]
 536         path = self.get_local_path(handler, query)
 537         file_path = os.path.normpath(path + '/' + f)
 538
 539         details = self.get_details_xml(tsn, file_path)
 540
 541         handler.send_xml(details)
 542
 543 class Video(BaseVideo, Pushable):
 544         pass
 545
 546 class VideoDetails(DictMixin):
 547
 548     def __init__(self, d=None):
 549         if d:
 550             self.d = d
 551         else:
 552             self.d = {}
 553
 554     def __getitem__(self, key):
 555         if key not in self.d:
 556             self.d[key] = self.default(key)
 557         return self.d[key]
 558
 559     def __contains__(self, key):
 560         return True
 561
 562     def __setitem__(self, key, value):
 563         self.d[key] = value
 564
 565     def __delitem__(self):
 566         del self.d[key]
 567
 568     def keys(self):
 569         return self.d.keys()
 570
 571     def __iter__(self):
 572         return self.d.__iter__()
 573
 574     def iteritems(self):
 575         return self.d.iteritems()
 576
 577     def default(self, key):
 578         defaults = {
 579             'showingBits' : '0',
 580             'displayMajorNumber' : '0',
 581             'displayMinorNumber' : '0',
 582             'isEpisode' : 'true',
 583             'colorCode' : ('COLOR', '4'),
 584             'showType' : ('SERIES', '5')
 585         }
 586         if key in defaults:
 587             return defaults[key]
 588         elif key.startswith('v'):
 589             return []
 590         else:
 591             return ''