yt_dlp/extractor/packtpub.py

   1 import json
   2
   3 from .common import InfoExtractor
   4 from ..networking.exceptions import HTTPError
   5 from ..utils import (
   6     ExtractorError,
   7     clean_html,
   8     # remove_end,
   9     str_or_none,
  10     strip_or_none,
  11     unified_timestamp,
  12 )
  13
  14
  15 class PacktPubBaseIE(InfoExtractor):
  16     # _PACKT_BASE = 'https://www.packtpub.com'
  17     _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
  18
  19
  20 class PacktPubIE(PacktPubBaseIE):
  21     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>[^/]+)/(?P<id>[^/]+)(?:/(?P<display_id>[^/?&#]+))?'
  22
  23     _TESTS = [{
  24         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  25         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  26         'info_dict': {
  27             'id': '20530',
  28             'ext': 'mp4',
  29             'title': 'Project Intro',
  30             'thumbnail': r're:(?i)^https?://.*\.jpg',
  31             'timestamp': 1490918400,
  32             'upload_date': '20170331',
  33         },
  34     }, {
  35         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  36         'only_matching': True,
  37     }, {
  38         'url': 'https://subscription.packtpub.com/video/programming/9781838988906/p1/video1_1/business-card-project',
  39         'only_matching': True,
  40     }]
  41     _NETRC_MACHINE = 'packtpub'
  42     _TOKEN = None
  43
  44     def _perform_login(self, username, password):
  45         try:
  46             self._TOKEN = self._download_json(
  47                 'https://services.packtpub.com/auth-v1/users/tokens', None,
  48                 'Downloading Authorization Token', data=json.dumps({
  49                     'username': username,
  50                     'password': password,
  51                 }).encode())['data']['access']
  52         except ExtractorError as e:
  53             if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 404):
  54                 message = self._parse_json(e.cause.response.read().decode(), None)['message']
  55                 raise ExtractorError(message, expected=True)
  56             raise
  57
  58     def _real_extract(self, url):
  59         course_id, chapter_id, video_id, display_id = self._match_valid_url(url).groups()
  60
  61         headers = {}
  62         if self._TOKEN:
  63             headers['Authorization'] = 'Bearer ' + self._TOKEN
  64         try:
  65             video_url = self._download_json(
  66                 f'https://services.packtpub.com/products-v1/products/{course_id}/{chapter_id}/{video_id}', video_id,
  67                 'Downloading JSON video', headers=headers)['data']
  68         except ExtractorError as e:
  69             if isinstance(e.cause, HTTPError) and e.cause.status == 400:
  70                 self.raise_login_required('This video is locked')
  71             raise
  72
  73         # TODO: find a better way to avoid duplicating course requests
  74         # metadata = self._download_json(
  75         #     '%s/products/%s/chapters/%s/sections/%s/metadata'
  76         #     % (self._MAPT_REST, course_id, chapter_id, video_id),
  77         #     video_id)['data']
  78
  79         # title = metadata['pageTitle']
  80         # course_title = metadata.get('title')
  81         # if course_title:
  82         #     title = remove_end(title, ' - %s' % course_title)
  83         # timestamp = unified_timestamp(metadata.get('publicationDate'))
  84         # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
  85
  86         return {
  87             'id': video_id,
  88             'url': video_url,
  89             'title': display_id or video_id,  # title,
  90             # 'thumbnail': thumbnail,
  91             # 'timestamp': timestamp,
  92         }
  93
  94
  95 class PacktPubCourseIE(PacktPubBaseIE):
  96     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
  97     _TESTS = [{
  98         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
  99         'info_dict': {
 100             'id': '9781787122215',
 101             'title': 'Learn Nodejs by building 12 projects [Video]',
 102             'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
 103         },
 104         'playlist_count': 90,
 105     }, {
 106         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 107         'only_matching': True,
 108     }]
 109
 110     @classmethod
 111     def suitable(cls, url):
 112         return False if PacktPubIE.suitable(url) else super().suitable(url)
 113
 114     def _real_extract(self, url):
 115         mobj = self._match_valid_url(url)
 116         url, course_id = mobj.group('url', 'id')
 117
 118         course = self._download_json(
 119             self._STATIC_PRODUCTS_BASE + f'{course_id}/toc', course_id)
 120         metadata = self._download_json(
 121             self._STATIC_PRODUCTS_BASE + f'{course_id}/summary',
 122             course_id, fatal=False) or {}
 123
 124         entries = []
 125         for chapter_num, chapter in enumerate(course['chapters'], 1):
 126             chapter_id = str_or_none(chapter.get('id'))
 127             sections = chapter.get('sections')
 128             if not chapter_id or not isinstance(sections, list):
 129                 continue
 130             chapter_info = {
 131                 'chapter': chapter.get('title'),
 132                 'chapter_number': chapter_num,
 133                 'chapter_id': chapter_id,
 134             }
 135             for section in sections:
 136                 section_id = str_or_none(section.get('id'))
 137                 if not section_id or section.get('contentType') != 'video':
 138                     continue
 139                 entry = {
 140                     '_type': 'url_transparent',
 141                     'url': '/'.join([url, chapter_id, section_id]),
 142                     'title': strip_or_none(section.get('title')),
 143                     'description': clean_html(section.get('summary')),
 144                     'thumbnail': metadata.get('coverImage'),
 145                     'timestamp': unified_timestamp(metadata.get('publicationDate')),
 146                     'ie_key': PacktPubIE.ie_key(),
 147                 }
 148                 entry.update(chapter_info)
 149                 entries.append(entry)
 150
 151         return self.playlist_result(
 152             entries, course_id, metadata.get('title'),
 153             clean_html(metadata.get('about')))