diff options
Diffstat (limited to 'hypervideo_dl/extractor/nebula.py')
| -rw-r--r-- | hypervideo_dl/extractor/nebula.py | 155 |
1 files changed, 57 insertions, 98 deletions
diff --git a/hypervideo_dl/extractor/nebula.py b/hypervideo_dl/extractor/nebula.py index 861fcb1..4f3e691 100644 --- a/hypervideo_dl/extractor/nebula.py +++ b/hypervideo_dl/extractor/nebula.py @@ -1,13 +1,11 @@ import itertools import json -import time -import urllib.error -import urllib.parse from .common import InfoExtractor -from ..utils import ExtractorError, parse_iso8601, try_get +from ..networking.exceptions import HTTPError +from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start -_BASE_URL_RE = r'https?://(?:www\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)' +_BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)' class NebulaBaseIE(InfoExtractor): @@ -15,11 +13,10 @@ class NebulaBaseIE(InfoExtractor): _nebula_api_token = None _nebula_bearer_token = None - _zype_access_token = None def _perform_nebula_auth(self, username, password): if not username or not password: - self.raise_login_required() + self.raise_login_required(method='password') data = json.dumps({'email': username, 'password': password}).encode('utf8') response = self._download_json( @@ -33,38 +30,10 @@ class NebulaBaseIE(InfoExtractor): note='Logging in to Nebula with supplied credentials', errnote='Authentication failed or rejected') if not response or not response.get('key'): - self.raise_login_required() - - # save nebula token as cookie - self._set_cookie( - 'nebula.app', 'nebula-auth', - urllib.parse.quote( - json.dumps({ - "apiToken": response["key"], - "isLoggingIn": False, - "isLoggingOut": False, - }, separators=(",", ":"))), - expire_time=int(time.time()) + 86400 * 365, - ) + self.raise_login_required(method='password') return response['key'] - def _retrieve_nebula_api_token(self, username=None, password=None): - """ - Check cookie jar for valid token. Try to authenticate using credentials if no valid token - can be found in the cookie jar. - """ - nebula_cookies = self._get_cookies('https://nebula.app') - nebula_cookie = nebula_cookies.get('nebula-auth') - if nebula_cookie: - self.to_screen('Authenticating to Nebula with token from cookie jar') - nebula_cookie_value = urllib.parse.unquote(nebula_cookie.value) - nebula_api_token = self._parse_json(nebula_cookie_value, None).get('apiToken') - if nebula_api_token: - return nebula_api_token - - return self._perform_nebula_auth(username, password) - def _call_nebula_api(self, url, video_id=None, method='GET', auth_type='api', note=''): assert method in ('GET', 'POST',) assert auth_type in ('api', 'bearer',) @@ -79,7 +48,7 @@ class NebulaBaseIE(InfoExtractor): return inner_call() except ExtractorError as exc: # if 401 or 403, attempt credential re-auth and retry - if exc.cause and isinstance(exc.cause, urllib.error.HTTPError) and exc.cause.code in (401, 403): + if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.status in (401, 403): self.to_screen(f'Reauthenticating to Nebula and retrying, because last {auth_type} call resulted in error {exc.cause.code}') self._perform_login() return inner_call() @@ -95,35 +64,25 @@ class NebulaBaseIE(InfoExtractor): note='Authorizing to Nebula') return response['token'] - def _fetch_zype_access_token(self): - """ - Get a Zype access token, which is required to access video streams -- in our case: to - generate video URLs. - """ - user_object = self._call_nebula_api('https://api.watchnebula.com/api/v1/auth/user/', note='Retrieving Zype access token') - - access_token = try_get(user_object, lambda x: x['zype_auth_info']['access_token'], str) - if not access_token: - if try_get(user_object, lambda x: x['is_subscribed'], bool): - # TODO: Reimplement the same Zype token polling the Nebula frontend implements - # see https://github.com/ytdl-org/youtube-dl/pull/24805#issuecomment-749231532 - raise ExtractorError( - 'Unable to extract Zype access token from Nebula API authentication endpoint. ' - 'Open an arbitrary video in a browser with this account to generate a token', - expected=True) - raise ExtractorError('Unable to extract Zype access token from Nebula API authentication endpoint') - return access_token + def _fetch_video_formats(self, slug): + stream_info = self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/stream/', + video_id=slug, + auth_type='bearer', + note='Fetching video stream info') + manifest_url = stream_info['manifest'] + return self._extract_m3u8_formats_and_subtitles(manifest_url, slug, 'mp4') def _build_video_info(self, episode): - zype_id = episode['zype_id'] - zype_video_url = f'https://player.zype.com/embed/{zype_id}.html?access_token={self._zype_access_token}' + fmts, subs = self._fetch_video_formats(episode['slug']) channel_slug = episode['channel_slug'] + channel_title = episode['channel_title'] + zype_id = episode.get('zype_id') return { - 'id': episode['zype_id'], + 'id': remove_start(episode['id'], 'video_episode:'), 'display_id': episode['slug'], - '_type': 'url_transparent', - 'ie_key': 'Zype', - 'url': zype_video_url, + 'formats': fmts, + 'subtitles': subs, + 'webpage_url': f'https://nebula.tv/{episode["slug"]}', 'title': episode['title'], 'description': episode['description'], 'timestamp': parse_iso8601(episode['published_at']), @@ -133,30 +92,32 @@ class NebulaBaseIE(InfoExtractor): 'height': key, } for key, tn in episode['assets']['thumbnail'].items()], 'duration': episode['duration'], - 'channel': episode['channel_title'], + 'channel': channel_title, 'channel_id': channel_slug, - 'channel_url': f'https://nebula.app/{channel_slug}', - 'uploader': episode['channel_title'], + 'channel_url': f'https://nebula.tv/{channel_slug}', + 'uploader': channel_title, 'uploader_id': channel_slug, - 'uploader_url': f'https://nebula.app/{channel_slug}', - 'series': episode['channel_title'], - 'creator': episode['channel_title'], + 'uploader_url': f'https://nebula.tv/{channel_slug}', + 'series': channel_title, + 'creator': channel_title, + 'extractor_key': NebulaIE.ie_key(), + 'extractor': NebulaIE.IE_NAME, + '_old_archive_ids': [make_archive_id(NebulaIE, zype_id)] if zype_id else None, } def _perform_login(self, username=None, password=None): - self._nebula_api_token = self._retrieve_nebula_api_token(username, password) + self._nebula_api_token = self._perform_nebula_auth(username, password) self._nebula_bearer_token = self._fetch_nebula_bearer_token() - self._zype_access_token = self._fetch_zype_access_token() class NebulaIE(NebulaBaseIE): _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P<id>[-\w]+)' _TESTS = [ { - 'url': 'https://nebula.app/videos/that-time-disney-remade-beauty-and-the-beast', + 'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast', 'md5': '14944cfee8c7beeea106320c47560efc', 'info_dict': { - 'id': '5c271b40b13fd613090034fd', + 'id': '84ed544d-4afd-4723-8cd5-2b95261f0abf', 'ext': 'mp4', 'title': 'That Time Disney Remade Beauty and the Beast', 'description': 'Note: this video was originally posted on YouTube with the sponsor read included. We weren’t able to remove it without reducing video quality, so it’s presented here in its original context.', @@ -167,47 +128,43 @@ class NebulaIE(NebulaBaseIE): 'uploader': 'Lindsay Ellis', 'uploader_id': 'lindsayellis', 'timestamp': 1533009600, - 'uploader_url': 'https://nebula.app/lindsayellis', + 'uploader_url': 'https://nebula.tv/lindsayellis', 'series': 'Lindsay Ellis', - 'average_rating': int, 'display_id': 'that-time-disney-remade-beauty-and-the-beast', - 'channel_url': 'https://nebula.app/lindsayellis', + 'channel_url': 'https://nebula.tv/lindsayellis', 'creator': 'Lindsay Ellis', 'duration': 2212, - 'view_count': int, 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', }, }, { - 'url': 'https://nebula.app/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore', + 'url': 'https://nebula.tv/videos/the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore', 'md5': 'd05739cf6c38c09322422f696b569c23', 'info_dict': { - 'id': '5e7e78171aaf320001fbd6be', + 'id': '7e623145-1b44-4ca3-aa0b-ed25a247ea34', 'ext': 'mp4', 'title': 'Landing Craft - How The Allies Got Ashore', 'description': r're:^In this episode we explore the unsung heroes of D-Day, the landing craft.', 'upload_date': '20200327', 'timestamp': 1585348140, - 'channel': 'Real Engineering', - 'channel_id': 'realengineering', - 'uploader': 'Real Engineering', - 'uploader_id': 'realengineering', - 'view_count': int, - 'series': 'Real Engineering', - 'average_rating': int, + 'channel': 'Real Engineering — The Logistics of D-Day', + 'channel_id': 'd-day', + 'uploader': 'Real Engineering — The Logistics of D-Day', + 'uploader_id': 'd-day', + 'series': 'Real Engineering — The Logistics of D-Day', 'display_id': 'the-logistics-of-d-day-landing-craft-how-the-allies-got-ashore', - 'creator': 'Real Engineering', + 'creator': 'Real Engineering — The Logistics of D-Day', 'duration': 841, - 'channel_url': 'https://nebula.app/realengineering', - 'uploader_url': 'https://nebula.app/realengineering', + 'channel_url': 'https://nebula.tv/d-day', + 'uploader_url': 'https://nebula.tv/d-day', 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', }, }, { - 'url': 'https://nebula.app/videos/money-episode-1-the-draw', + 'url': 'https://nebula.tv/videos/money-episode-1-the-draw', 'md5': 'ebe28a7ad822b9ee172387d860487868', 'info_dict': { - 'id': '5e779ebdd157bc0001d1c75a', + 'id': 'b96c5714-9e2b-4ec3-b3f1-20f6e89cc553', 'ext': 'mp4', 'title': 'Episode 1: The Draw', 'description': r'contains:There’s free money on offer… if the players can all work together.', @@ -217,14 +174,12 @@ class NebulaIE(NebulaBaseIE): 'channel_id': 'tom-scott-presents-money', 'uploader': 'Tom Scott Presents: Money', 'uploader_id': 'tom-scott-presents-money', - 'uploader_url': 'https://nebula.app/tom-scott-presents-money', + 'uploader_url': 'https://nebula.tv/tom-scott-presents-money', 'duration': 825, - 'channel_url': 'https://nebula.app/tom-scott-presents-money', - 'view_count': int, + 'channel_url': 'https://nebula.tv/tom-scott-presents-money', 'series': 'Tom Scott Presents: Money', 'display_id': 'money-episode-1-the-draw', 'thumbnail': r're:https://\w+\.cloudfront\.net/[\w-]+\.jpeg?.*', - 'average_rating': int, 'creator': 'Tom Scott Presents: Money', }, }, @@ -232,10 +187,14 @@ class NebulaIE(NebulaBaseIE): 'url': 'https://watchnebula.com/videos/money-episode-1-the-draw', 'only_matching': True, }, + { + 'url': 'https://beta.nebula.tv/videos/money-episode-1-the-draw', + 'only_matching': True, + }, ] def _fetch_video_metadata(self, slug): - return self._call_nebula_api(f'https://content.watchnebula.com/video/{slug}/', + return self._call_nebula_api(f'https://content.api.nebula.app/video/{slug}/', video_id=slug, auth_type='bearer', note='Fetching video meta data') @@ -251,7 +210,7 @@ class NebulaSubscriptionsIE(NebulaBaseIE): _VALID_URL = rf'{_BASE_URL_RE}/myshows' _TESTS = [ { - 'url': 'https://nebula.app/myshows', + 'url': 'https://nebula.tv/myshows', 'playlist_mincount': 1, 'info_dict': { 'id': 'myshows', @@ -279,7 +238,7 @@ class NebulaChannelIE(NebulaBaseIE): _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|videos/)(?P<id>[-\w]+)' _TESTS = [ { - 'url': 'https://nebula.app/tom-scott-presents-money', + 'url': 'https://nebula.tv/tom-scott-presents-money', 'info_dict': { 'id': 'tom-scott-presents-money', 'title': 'Tom Scott Presents: Money', @@ -287,13 +246,13 @@ class NebulaChannelIE(NebulaBaseIE): }, 'playlist_count': 5, }, { - 'url': 'https://nebula.app/lindsayellis', + 'url': 'https://nebula.tv/lindsayellis', 'info_dict': { 'id': 'lindsayellis', 'title': 'Lindsay Ellis', 'description': 'Enjoy these hottest of takes on Disney, Transformers, and Musicals.', }, - 'playlist_mincount': 100, + 'playlist_mincount': 2, }, ] |