yt_dlp/extractor/mediasite.py

   1 import json
   2 import re
   3 import urllib.parse
   4
   5 from .common import InfoExtractor
   6 from ..utils import (
   7     ExtractorError,
   8     float_or_none,
   9     mimetype2ext,
  10     smuggle_url,
  11     str_or_none,
  12     try_call,
  13     try_get,
  14     unsmuggle_url,
  15     url_or_none,
  16     urljoin,
  17 )
  18 from ..utils.traversal import traverse_obj
  19
  20 _ID_RE = r'(?:[0-9a-f]{32,34}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12,14})'
  21
  22
  23 class MediasiteIE(InfoExtractor):
  24     _VALID_URL = rf'(?xi)https?://[^/]+/Mediasite/(?:Play|Showcase/[^/#?]+/Presentation)/(?P<id>{_ID_RE})(?P<query>\?[^#]+|)'
  25     _EMBED_REGEX = [rf'(?xi)<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:(?:https?:)?//[^/]+)?/Mediasite/Play/{_ID_RE}(?:\?.*?)?)\1']
  26     _TESTS = [
  27         {
  28             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271681e4f199af3c60d1f82869b1d',
  29             'info_dict': {
  30                 'id': '2db6c271681e4f199af3c60d1f82869b1d',
  31                 'ext': 'mp4',
  32                 'title': 'Lecture: Tuesday, September 20, 2016 - Sir Andrew Wiles',
  33                 'description': 'Sir Andrew Wiles: “Equations in arithmetic”\\n\\nI will describe some of the interactions between modern number theory and the problem of solving equations in rational numbers or integers\\u0027.',
  34                 'timestamp': 1474268400.0,
  35                 'upload_date': '20160919',
  36             },
  37         },
  38         {
  39             'url': 'http://mediasite.uib.no/Mediasite/Play/90bb363295d945d6b548c867d01181361d?catalog=a452b7df-9ae1-46b7-a3ba-aceeb285f3eb',
  40             'info_dict': {
  41                 'id': '90bb363295d945d6b548c867d01181361d',
  42                 'ext': 'mp4',
  43                 'upload_date': '20150429',
  44                 'title': '5) IT-forum 2015-Dag 1  - Dungbeetle -  How and why Rain created a tiny bug tracker for Unity',
  45                 'timestamp': 1430311380.0,
  46             },
  47         },
  48         {
  49             'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
  50             'md5': '481fda1c11f67588c0d9d8fbdced4e39',
  51             'info_dict': {
  52                 'id': '585a43626e544bdd97aeb71a0ec907a01d',
  53                 'ext': 'mp4',
  54                 'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
  55                 'description': '',
  56                 'thumbnail': r're:^https?://.*\.jpg(?:\?.*)?$',
  57                 'duration': 7713.088,
  58                 'timestamp': 1413309600,
  59                 'upload_date': '20141014',
  60             },
  61         },
  62         {
  63             'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
  64             'md5': 'ef1fdded95bdf19b12c5999949419c92',
  65             'info_dict': {
  66                 'id': '86a9ea9f53e149079fbdb4202b521ed21d',
  67                 'ext': 'wmv',
  68                 'title': '64ste Vakantiecursus: Afvalwater',
  69                 'description': 'md5:7fd774865cc69d972f542b157c328305',
  70                 'thumbnail': r're:^https?://.*\.jpg(?:\?.*?)?$',
  71                 'duration': 10853,
  72                 'timestamp': 1326446400,
  73                 'upload_date': '20120113',
  74             },
  75         },
  76         {
  77             'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
  78             'md5': '9422edc9b9a60151727e4b6d8bef393d',
  79             'info_dict': {
  80                 'id': '24aace4429fc450fb5b38cdbf424a66e1d',
  81                 'ext': 'mp4',
  82                 'title': 'Xyce Software Training - Section 1',
  83                 'description': r're:(?s)SAND Number: SAND 2013-7800.{200,}',
  84                 'upload_date': '20120409',
  85                 'timestamp': 1333983600,
  86                 'duration': 7794,
  87             },
  88         },
  89         {
  90             'url': 'https://collegerama.tudelft.nl/Mediasite/Showcase/livebroadcast/Presentation/ada7020854f743c49fbb45c9ec7dbb351d',
  91             'only_matching': True,
  92         },
  93         {
  94             'url': 'https://mediasite.ntnu.no/Mediasite/Showcase/default/Presentation/7d8b913259334b688986e970fae6fcb31d',
  95             'only_matching': True,
  96         },
  97         {
  98             # dashed id
  99             'url': 'https://hitsmediaweb.h-its.org/mediasite/Play/2db6c271-681e-4f19-9af3-c60d1f82869b1d',
 100             'only_matching': True,
 101         },
 102     ]
 103
 104     # look in Mediasite.Core.js (Mediasite.ContentStreamType[*])
 105     _STREAM_TYPES = {
 106         0: 'video1',  # the main video
 107         2: 'slide',
 108         3: 'presentation',
 109         4: 'video2',  # screencast?
 110         5: 'video3',
 111     }
 112
 113     @classmethod
 114     def _extract_embed_urls(cls, url, webpage):
 115         for embed_url in super()._extract_embed_urls(url, webpage):
 116             yield smuggle_url(embed_url, {'UrlReferrer': url})
 117
 118     def __extract_slides(self, *, stream_id, snum, stream, duration, images):
 119         slide_base_url = stream['SlideBaseUrl']
 120
 121         fname_template = stream['SlideImageFileNameTemplate']
 122         if fname_template != 'slide_{0:D4}.jpg':
 123             self.report_warning('Unusual slide file name template; report a bug if slide downloading fails')
 124         fname_template = re.sub(r'\{0:D([0-9]+)\}', r'{0:0\1}', fname_template)
 125
 126         fragments = []
 127         for i, slide in enumerate(stream['Slides']):
 128             if i == 0:
 129                 if slide['Time'] > 0:
 130                     default_slide = images.get('DefaultSlide')
 131                     if default_slide is None:
 132                         default_slide = images.get('DefaultStreamImage')
 133                     if default_slide is not None:
 134                         default_slide = default_slide['ImageFilename']
 135                     if default_slide is not None:
 136                         fragments.append({
 137                             'path': default_slide,
 138                             'duration': slide['Time'] / 1000,
 139                         })
 140
 141             next_time = try_call(
 142                 lambda: stream['Slides'][i + 1]['Time'],
 143                 lambda: duration,
 144                 lambda: slide['Time'],
 145                 expected_type=(int, float))
 146
 147             fragments.append({
 148                 'path': fname_template.format(slide.get('Number', i + 1)),
 149                 'duration': (next_time - slide['Time']) / 1000,
 150             })
 151
 152         return {
 153             'format_id': f'{stream_id}-{snum}.slides',
 154             'ext': 'mhtml',
 155             'url': slide_base_url,
 156             'protocol': 'mhtml',
 157             'acodec': 'none',
 158             'vcodec': 'none',
 159             'format_note': 'Slides',
 160             'fragments': fragments,
 161             'fragment_base_url': slide_base_url,
 162         }
 163
 164     def _real_extract(self, url):
 165         url, data = unsmuggle_url(url, {})
 166         mobj = self._match_valid_url(url)
 167         resource_id = mobj.group('id')
 168         query = mobj.group('query')
 169
 170         webpage, urlh = self._download_webpage_handle(url, resource_id)  # XXX: add UrlReferrer?
 171         redirect_url = urlh.url
 172
 173         # XXX: might have also extracted UrlReferrer and QueryString from the html
 174         service_path = urllib.parse.urljoin(redirect_url, self._html_search_regex(
 175             r'<div[^>]+\bid=["\']ServicePath[^>]+>(.+?)</div>', webpage, resource_id,
 176             default='/Mediasite/PlayerService/PlayerService.svc/json'))
 177
 178         player_options = self._download_json(
 179             f'{service_path}/GetPlayerOptions', resource_id,
 180             headers={
 181                 'Content-type': 'application/json; charset=utf-8',
 182                 'X-Requested-With': 'XMLHttpRequest',
 183             },
 184             data=json.dumps({
 185                 'getPlayerOptionsRequest': {
 186                     'ResourceId': resource_id,
 187                     'QueryString': query,
 188                     'UrlReferrer': data.get('UrlReferrer', ''),
 189                     'UseScreenReader': False,
 190                 },
 191             }).encode())['d']
 192
 193         presentation = player_options['Presentation']
 194         title = presentation['Title']
 195
 196         if presentation is None:
 197             raise ExtractorError(
 198                 'Mediasite says: {}'.format(player_options['PlayerPresentationStatusMessage']),
 199                 expected=True)
 200
 201         thumbnails = []
 202         formats = []
 203         for snum, stream in enumerate(presentation['Streams']):
 204             stream_type = stream.get('StreamType')
 205             if stream_type is None:
 206                 continue
 207
 208             video_urls = stream.get('VideoUrls')
 209             if not isinstance(video_urls, list):
 210                 video_urls = []
 211
 212             stream_id = self._STREAM_TYPES.get(
 213                 stream_type, 'type%u' % stream_type)
 214
 215             stream_formats = []
 216             for unum, video in enumerate(video_urls):
 217                 video_url = url_or_none(video.get('Location'))
 218                 if not video_url:
 219                     continue
 220                 # XXX: if Stream.get('CanChangeScheme', False), switch scheme to HTTP/HTTPS
 221
 222                 media_type = video.get('MediaType')
 223                 ext = mimetype2ext(video.get('MimeType'))
 224                 if media_type == 'SS':
 225                     stream_formats.extend(self._extract_ism_formats(
 226                         video_url, resource_id,
 227                         ism_id=f'{stream_id}-{snum}.{unum}',
 228                         fatal=False))
 229                 elif media_type == 'Dash':
 230                     stream_formats.extend(self._extract_mpd_formats(
 231                         video_url, resource_id,
 232                         mpd_id=f'{stream_id}-{snum}.{unum}',
 233                         fatal=False))
 234                 elif ext in ('m3u', 'm3u8'):
 235                     stream_formats.extend(self._extract_m3u8_formats(
 236                         video_url, resource_id,
 237                         m3u8_id=f'{stream_id}-{snum}.{unum}',
 238                         fatal=False))
 239                 else:
 240                     stream_formats.append({
 241                         'format_id': f'{stream_id}-{snum}.{unum}',
 242                         'url': video_url,
 243                         'ext': ext,
 244                     })
 245
 246             images = traverse_obj(player_options, ('PlayerLayoutOptions', 'Images', {dict}))
 247             if stream.get('HasSlideContent') and images:
 248                 stream_formats.append(self.__extract_slides(
 249                     stream_id=stream_id,
 250                     snum=snum,
 251                     stream=stream,
 252                     duration=presentation.get('Duration'),
 253                     images=images,
 254                 ))
 255
 256             # disprefer 'secondary' streams
 257             if stream_type != 0:
 258                 for fmt in stream_formats:
 259                     fmt['quality'] = -10
 260
 261             thumbnail_url = stream.get('ThumbnailUrl')
 262             if thumbnail_url:
 263                 thumbnails.append({
 264                     'id': f'{stream_id}-{snum}',
 265                     'url': urljoin(redirect_url, thumbnail_url),
 266                     'preference': -1 if stream_type != 0 else 0,
 267                 })
 268             formats.extend(stream_formats)
 269
 270         # XXX: Presentation['Presenters']
 271         # XXX: Presentation['Transcript']
 272
 273         return {
 274             'id': resource_id,
 275             'title': title,
 276             'description': presentation.get('Description'),
 277             'duration': float_or_none(presentation.get('Duration'), 1000),
 278             'timestamp': float_or_none(presentation.get('UnixTime'), 1000),
 279             'formats': formats,
 280             'thumbnails': thumbnails,
 281         }
 282
 283
 284 class MediasiteCatalogIE(InfoExtractor):
 285     _VALID_URL = rf'''(?xi)
 286                         (?P<url>https?://[^/]+/Mediasite)
 287                         /Catalog/Full/
 288                         (?P<catalog_id>{_ID_RE})
 289                         (?:
 290                             /(?P<current_folder_id>{_ID_RE})
 291                             /(?P<root_dynamic_folder_id>{_ID_RE})
 292                         )?
 293                     '''
 294     _TESTS = [{
 295         'url': 'http://events7.mediasite.com/Mediasite/Catalog/Full/631f9e48530d454381549f955d08c75e21',
 296         'info_dict': {
 297             'id': '631f9e48530d454381549f955d08c75e21',
 298             'title': 'WCET Summit: Adaptive Learning in Higher Ed: Improving Outcomes Dynamically',
 299         },
 300         'playlist_count': 6,
 301         'expected_warnings': ['is not a supported codec'],
 302     }, {
 303         # with CurrentFolderId and RootDynamicFolderId
 304         'url': 'https://medaudio.medicine.iu.edu/Mediasite/Catalog/Full/9518c4a6c5cf4993b21cbd53e828a92521/97a9db45f7ab47428c77cd2ed74bb98f14/9518c4a6c5cf4993b21cbd53e828a92521',
 305         'info_dict': {
 306             'id': '9518c4a6c5cf4993b21cbd53e828a92521',
 307             'title': 'IUSM Family and Friends Sessions',
 308         },
 309         'playlist_count': 2,
 310     }, {
 311         'url': 'http://uipsyc.mediasite.com/mediasite/Catalog/Full/d5d79287c75243c58c50fef50174ec1b21',
 312         'only_matching': True,
 313     }, {
 314         # no AntiForgeryToken
 315         'url': 'https://live.libraries.psu.edu/Mediasite/Catalog/Full/8376d4b24dd1457ea3bfe4cf9163feda21',
 316         'only_matching': True,
 317     }, {
 318         'url': 'https://medaudio.medicine.iu.edu/Mediasite/Catalog/Full/9518c4a6c5cf4993b21cbd53e828a92521/97a9db45f7ab47428c77cd2ed74bb98f14/9518c4a6c5cf4993b21cbd53e828a92521',
 319         'only_matching': True,
 320     }, {
 321         # dashed id
 322         'url': 'http://events7.mediasite.com/Mediasite/Catalog/Full/631f9e48-530d-4543-8154-9f955d08c75e',
 323         'only_matching': True,
 324     }]
 325
 326     def _real_extract(self, url):
 327         mobj = self._match_valid_url(url)
 328         mediasite_url = mobj.group('url')
 329         catalog_id = mobj.group('catalog_id')
 330         current_folder_id = mobj.group('current_folder_id') or catalog_id
 331         root_dynamic_folder_id = mobj.group('root_dynamic_folder_id')
 332
 333         webpage = self._download_webpage(url, catalog_id)
 334
 335         # AntiForgeryToken is optional (e.g. [1])
 336         # 1. https://live.libraries.psu.edu/Mediasite/Catalog/Full/8376d4b24dd1457ea3bfe4cf9163feda21
 337         anti_forgery_token = self._search_regex(
 338             r'AntiForgeryToken\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
 339             webpage, 'anti forgery token', default=None, group='value')
 340         if anti_forgery_token:
 341             anti_forgery_header = self._search_regex(
 342                 r'AntiForgeryHeaderName\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
 343                 webpage, 'anti forgery header name',
 344                 default='X-SOFO-AntiForgeryHeader', group='value')
 345
 346         data = {
 347             'IsViewPage': True,
 348             'IsNewFolder': True,
 349             'AuthTicket': None,
 350             'CatalogId': catalog_id,
 351             'CurrentFolderId': current_folder_id,
 352             'RootDynamicFolderId': root_dynamic_folder_id,
 353             'ItemsPerPage': 1000,
 354             'PageIndex': 0,
 355             'PermissionMask': 'Execute',
 356             'CatalogSearchType': 'SearchInFolder',
 357             'SortBy': 'Date',
 358             'SortDirection': 'Descending',
 359             'StartDate': None,
 360             'EndDate': None,
 361             'StatusFilterList': None,
 362             'PreviewKey': None,
 363             'Tags': [],
 364         }
 365
 366         headers = {
 367             'Content-Type': 'application/json; charset=UTF-8',
 368             'Referer': url,
 369             'X-Requested-With': 'XMLHttpRequest',
 370         }
 371         if anti_forgery_token:
 372             headers[anti_forgery_header] = anti_forgery_token
 373
 374         catalog = self._download_json(
 375             f'{mediasite_url}/Catalog/Data/GetPresentationsForFolder',
 376             catalog_id, data=json.dumps(data).encode(), headers=headers)
 377
 378         entries = []
 379         for video in catalog['PresentationDetailsList']:
 380             if not isinstance(video, dict):
 381                 continue
 382             video_id = str_or_none(video.get('Id'))
 383             if not video_id:
 384                 continue
 385             entries.append(self.url_result(
 386                 f'{mediasite_url}/Play/{video_id}',
 387                 ie=MediasiteIE.ie_key(), video_id=video_id))
 388
 389         title = try_get(
 390             catalog, lambda x: x['CurrentFolder']['Name'], str)
 391
 392         return self.playlist_result(entries, catalog_id, title)
 393
 394
 395 class MediasiteNamedCatalogIE(InfoExtractor):
 396     _VALID_URL = r'(?xi)(?P<url>https?://[^/]+/Mediasite)/Catalog/catalogs/(?P<catalog_name>[^/?#&]+)'
 397     _TESTS = [{
 398         'url': 'https://msite.misis.ru/Mediasite/Catalog/catalogs/2016-industrial-management-skriabin-o-o',
 399         'only_matching': True,
 400     }]
 401
 402     def _real_extract(self, url):
 403         mobj = self._match_valid_url(url)
 404         mediasite_url = mobj.group('url')
 405         catalog_name = mobj.group('catalog_name')
 406
 407         webpage = self._download_webpage(url, catalog_name)
 408
 409         catalog_id = self._search_regex(
 410             rf'CatalogId\s*:\s*["\']({_ID_RE})', webpage, 'catalog id')
 411
 412         return self.url_result(
 413             f'{mediasite_url}/Catalog/Full/{catalog_id}',
 414             ie=MediasiteCatalogIE.ie_key(), video_id=catalog_id)