yt_dlp/extractor/viidea.py

   1 import re
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from ..networking.exceptions import HTTPError
   6 from ..utils import (
   7     ExtractorError,
   8     js_to_json,
   9     parse_duration,
  10     parse_iso8601,
  11 )
  12
  13
  14 class ViideaIE(InfoExtractor):
  15     _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
  16             videolectures\.net|
  17             flexilearn\.viidea\.net|
  18             presentations\.ocwconsortium\.org|
  19             video\.travel-zoom\.si|
  20             video\.pomp-forum\.si|
  21             tv\.nil\.si|
  22             video\.hekovnik.com|
  23             video\.szko\.si|
  24             kpk\.viidea\.com|
  25             inside\.viidea\.net|
  26             video\.kiberpipa\.org|
  27             bvvideo\.si|
  28             kongres\.viidea\.net|
  29             edemokracija\.viidea\.com
  30         )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''
  31
  32     _TESTS = [{
  33         'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
  34         'info_dict': {
  35             'id': '20171',
  36             'display_id': 'promogram_igor_mekjavic_eng',
  37             'ext': 'mp4',
  38             'title': 'Automatics, robotics and biocybernetics',
  39             'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
  40             'thumbnail': r're:http://.*\.jpg',
  41             'timestamp': 1372349289,
  42             'upload_date': '20130627',
  43             'duration': 565,
  44         },
  45         'params': {
  46             # m3u8 download
  47             'skip_download': True,
  48         },
  49     }, {
  50         # video with invalid direct format links (HTTP 403)
  51         'url': 'http://videolectures.net/russir2010_filippova_nlp/',
  52         'info_dict': {
  53             'id': '14891',
  54             'display_id': 'russir2010_filippova_nlp',
  55             'ext': 'flv',
  56             'title': 'NLP at Google',
  57             'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
  58             'thumbnail': r're:http://.*\.jpg',
  59             'timestamp': 1284375600,
  60             'upload_date': '20100913',
  61             'duration': 5352,
  62         },
  63         'params': {
  64             # rtmp download
  65             'skip_download': True,
  66         },
  67     }, {
  68         # event playlist
  69         'url': 'http://videolectures.net/deeplearning2015_montreal/',
  70         'info_dict': {
  71             'id': '23181',
  72             'title': 'Deep Learning Summer School, Montreal 2015',
  73             'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
  74             'thumbnail': r're:http://.*\.jpg',
  75             'timestamp': 1438560000,
  76         },
  77         'playlist_count': 30,
  78     }, {
  79         # multi part lecture
  80         'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
  81         'info_dict': {
  82             'id': '9737',
  83             'display_id': 'mlss09uk_bishop_ibi',
  84             'title': 'Introduction To Bayesian Inference',
  85             'thumbnail': r're:http://.*\.jpg',
  86             'timestamp': 1251622800,
  87         },
  88         'playlist': [{
  89             'info_dict': {
  90                 'id': '9737_part1',
  91                 'display_id': 'mlss09uk_bishop_ibi_part1',
  92                 'ext': 'wmv',
  93                 'title': 'Introduction To Bayesian Inference (Part 1)',
  94                 'thumbnail': r're:http://.*\.jpg',
  95                 'duration': 4622,
  96                 'timestamp': 1251622800,
  97                 'upload_date': '20090830',
  98             },
  99         }, {
 100             'info_dict': {
 101                 'id': '9737_part2',
 102                 'display_id': 'mlss09uk_bishop_ibi_part2',
 103                 'ext': 'wmv',
 104                 'title': 'Introduction To Bayesian Inference (Part 2)',
 105                 'thumbnail': r're:http://.*\.jpg',
 106                 'duration': 5641,
 107                 'timestamp': 1251622800,
 108                 'upload_date': '20090830',
 109             },
 110         }],
 111         'playlist_count': 2,
 112     }]
 113
 114     def _real_extract(self, url):
 115         lecture_slug, explicit_part_id = self._match_valid_url(url).groups()
 116
 117         webpage = self._download_webpage(url, lecture_slug)
 118
 119         cfg = self._parse_json(self._search_regex(
 120             [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
 121              r'cfg\s*:\s*({[^}]+})'],
 122             webpage, 'cfg'), lecture_slug, js_to_json)
 123
 124         lecture_id = str(cfg['obj_id'])
 125
 126         base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
 127
 128         try:
 129             lecture_data = self._download_json(
 130                 f'{base_url}/site/api/lecture/{lecture_id}?format=json',
 131                 lecture_id)['lecture'][0]
 132         except ExtractorError as e:
 133             if isinstance(e.cause, HTTPError) and e.cause.status == 403:
 134                 msg = self._parse_json(
 135                     e.cause.response.read().decode('utf-8'), lecture_id)
 136                 raise ExtractorError(msg['detail'], expected=True)
 137             raise
 138
 139         lecture_info = {
 140             'id': lecture_id,
 141             'display_id': lecture_slug,
 142             'title': lecture_data['title'],
 143             'timestamp': parse_iso8601(lecture_data.get('time')),
 144             'description': lecture_data.get('description_wiki'),
 145             'thumbnail': lecture_data.get('thumb'),
 146         }
 147
 148         playlist_entries = []
 149         lecture_type = lecture_data.get('type')
 150         parts = [str(video) for video in cfg.get('videos', [])]
 151         if parts:
 152             multipart = len(parts) > 1
 153
 154             def extract_part(part_id):
 155                 smil_url = f'{base_url}/{lecture_slug}/video/{part_id}/smil.xml'
 156                 smil = self._download_smil(smil_url, lecture_id)
 157                 info = self._parse_smil(smil, smil_url, lecture_id)
 158                 info['id'] = lecture_id if not multipart else f'{lecture_id}_part{part_id}'
 159                 info['display_id'] = lecture_slug if not multipart else f'{lecture_slug}_part{part_id}'
 160                 if multipart:
 161                     info['title'] += f' (Part {part_id})'
 162                 switch = smil.find('.//switch')
 163                 if switch is not None:
 164                     info['duration'] = parse_duration(switch.attrib.get('dur'))
 165                 item_info = lecture_info.copy()
 166                 item_info.update(info)
 167                 return item_info
 168
 169             if explicit_part_id or not multipart:
 170                 result = extract_part(explicit_part_id or parts[0])
 171             else:
 172                 result = {
 173                     '_type': 'multi_video',
 174                     'entries': [extract_part(part) for part in parts],
 175                 }
 176                 result.update(lecture_info)
 177
 178             # Immediately return explicitly requested part or non event item
 179             if explicit_part_id or lecture_type != 'evt':
 180                 return result
 181
 182             playlist_entries.append(result)
 183
 184         # It's probably a playlist
 185         if not parts or lecture_type == 'evt':
 186             playlist_webpage = self._download_webpage(
 187                 f'{base_url}/site/ajax/drilldown/?id={lecture_id}', lecture_id)
 188             entries = [
 189                 self.url_result(urllib.parse.urljoin(url, video_url), 'Viidea')
 190                 for _, video_url in re.findall(
 191                     r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
 192             playlist_entries.extend(entries)
 193
 194         playlist = self.playlist_result(playlist_entries, lecture_id)
 195         playlist.update(lecture_info)
 196         return playlist