From f69b0554eb4500f1bdd0e07484d6b0a91e2b050c Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 9 Dec 2022 23:25:37 +0000 Subject: [extractor/slideslive] Fix extractor (#5737) Closes #1532 Authored by: bashonly, Grub4K --- yt_dlp/extractor/slideslive.py | 163 +++++++++++++++++++++++++++++++---------- 1 file changed, 124 insertions(+), 39 deletions(-) (limited to 'yt_dlp/extractor/slideslive.py') diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py index 9a60a79e7..86c26a8a2 100644 --- a/yt_dlp/extractor/slideslive.py +++ b/yt_dlp/extractor/slideslive.py @@ -1,92 +1,176 @@ from .common import InfoExtractor from ..utils import ( - bool_or_none, smuggle_url, - try_get, + traverse_obj, + unified_timestamp, url_or_none, ) class SlidesLiveIE(InfoExtractor): _VALID_URL = r'https?://slideslive\.com/(?P[0-9]+)' - _WORKING = False _TESTS = [{ - # video_service_name = YOUTUBE + # service_name = yoda 'url': 'https://slideslive.com/38902413/gcc-ia16-backend', - 'md5': 'b29fcd6c6952d0c79c5079b0e7a07e6f', 'info_dict': { - 'id': 'LMtgR8ba0b0', + 'id': '38902413', 'ext': 'mp4', 'title': 'GCC IA16 backend', - 'description': 'Watch full version of this video at https://slideslive.com/38902413.', - 'uploader': 'SlidesLive Videos - A', - 'uploader_id': 'UC62SdArr41t_-_fX40QCLRw', - 'timestamp': 1597615266, - 'upload_date': '20170925', - } + 'timestamp': 1648189972, + 'upload_date': '20220325', + 'thumbnail': r're:^https?://.*\.jpg', + }, + 'params': { + 'skip_download': 'm3u8', + }, }, { - # video_service_name = yoda + # service_name = yoda 'url': 'https://slideslive.com/38935785', - 'md5': '575cd7a6c0acc6e28422fe76dd4bcb1a', 'info_dict': { - 'id': 'RMraDYN5ozA_', + 'id': '38935785', 'ext': 'mp4', 'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges', + 'upload_date': '20211115', + 'timestamp': 1636996003, + 'thumbnail': r're:^https?://.*\.jpg', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # service_name = yoda + 'url': 'https://slideslive.com/38973182/how-should-a-machine-learning-researcher-think-about-ai-ethics', + 'info_dict': { + 'id': '38973182', + 'ext': 'mp4', + 'title': 'How Should a Machine Learning Researcher Think About AI Ethics?', + 'upload_date': '20220201', + 'thumbnail': r're:^https?://.*\.jpg', + 'timestamp': 1643728135, + }, + 'params': { + 'skip_download': 'm3u8', }, }, { - # video_service_name = youtube + # service_name = youtube + 'url': 'https://slideslive.com/38897546/special-metaprednaska-petra-ludwiga-hodnoty-pro-lepsi-spolecnost', + 'md5': '8a79b5e3d700837f40bd2afca3c8fa01', + 'info_dict': { + 'id': 'jmg02wCJD5M', + 'display_id': '38897546', + 'ext': 'mp4', + 'title': 'SPECIÁL: Meta-přednáška Petra Ludwiga - Hodnoty pro lepší společnost', + 'description': 'Watch full version of this video at https://slideslive.com/38897546.', + 'channel_url': 'https://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw', + 'channel': 'SlidesLive Videos - G1', + 'channel_id': 'UCZWdAkNYFncuX0khyvhqnxw', + 'uploader_id': 'UCZWdAkNYFncuX0khyvhqnxw', + 'uploader': 'SlidesLive Videos - G1', + 'uploader_url': 'http://www.youtube.com/channel/UCZWdAkNYFncuX0khyvhqnxw', + 'live_status': 'not_live', + 'upload_date': '20160710', + 'timestamp': 1618786715, + 'duration': 6827, + 'like_count': int, + 'view_count': int, + 'comment_count': int, + 'channel_follower_count': int, + 'age_limit': 0, + 'thumbnail': r're:^https?://.*\.jpg', + 'playable_in_embed': True, + 'availability': 'unlisted', + 'tags': [], + 'categories': ['People & Blogs'], + }, + }, { + # service_name = youtube 'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend', 'only_matching': True, }, { - # video_service_name = url + # service_name = url 'url': 'https://slideslive.com/38922070/learning-transferable-skills-1', 'only_matching': True, }, { - # video_service_name = vimeo + # service_name = vimeo 'url': 'https://slideslive.com/38921896/retrospectives-a-venue-for-selfreflection-in-ml-research-3', 'only_matching': True, }] + def _extract_custom_m3u8_info(self, m3u8_data): + m3u8_dict = {} + + lookup = { + 'PRESENTATION-TITLE': 'title', + 'PRESENTATION-UPDATED-AT': 'timestamp', + 'PRESENTATION-THUMBNAIL': 'thumbnail', + 'PLAYLIST-TYPE': 'playlist_type', + 'VOD-VIDEO-SERVICE-NAME': 'service_name', + 'VOD-VIDEO-ID': 'service_id', + 'VOD-VIDEO-SERVERS': 'video_servers', + 'VOD-SUBTITLES': 'subtitles', + } + + for line in m3u8_data.splitlines(): + if not line.startswith('#EXT-SL-'): + continue + tag, _, value = line.partition(':') + key = lookup.get(tag.lstrip('#EXT-SL-')) + if not key: + continue + m3u8_dict[key] = value + + # Some values are stringified JSON arrays + for key in ('video_servers', 'subtitles'): + if key in m3u8_dict: + m3u8_dict[key] = self._parse_json(m3u8_dict[key], None, fatal=False) or [] + + return m3u8_dict + def _real_extract(self, url): video_id = self._match_id(url) - video_data = self._download_json( - 'https://ben.slideslive.com/player/' + video_id, video_id) - service_name = video_data['video_service_name'].lower() + webpage = self._download_webpage(url, video_id) + player_token = self._search_regex(r'data-player-token="([^"]+)"', webpage, 'player token') + player_data = self._download_webpage( + f'https://ben.slideslive.com/player/{video_id}', video_id, + note='Downloading player info', query={'player_token': player_token}) + player_info = self._extract_custom_m3u8_info(player_data) + + service_name = player_info['service_name'].lower() assert service_name in ('url', 'yoda', 'vimeo', 'youtube') - service_id = video_data['video_service_id'] + service_id = player_info['service_id'] + subtitles = {} - for sub in try_get(video_data, lambda x: x['subtitles'], list) or []: - if not isinstance(sub, dict): - continue + for sub in traverse_obj(player_info, ('subtitles', ...), expected_type=dict): webvtt_url = url_or_none(sub.get('webvtt_url')) if not webvtt_url: continue - lang = sub.get('language') or 'en' - subtitles.setdefault(lang, []).append({ + subtitles.setdefault(sub.get('language') or 'en', []).append({ 'url': webvtt_url, + 'ext': 'vtt', }) + info = { 'id': video_id, - 'thumbnail': video_data.get('thumbnail'), - 'is_live': bool_or_none(video_data.get('is_live')), + 'title': player_info.get('title') or self._html_search_meta('title', webpage, default=''), + 'timestamp': unified_timestamp(player_info.get('timestamp')), + 'is_live': player_info.get('playlist_type') != 'vod', + 'thumbnail': url_or_none(player_info.get('thumbnail')), 'subtitles': subtitles, } + if service_name in ('url', 'yoda'): - info['title'] = video_data['title'] if service_name == 'url': info['url'] = service_id else: + cdn_hostname = player_info['video_servers'][0] formats = [] - _MANIFEST_PATTERN = 'https://01.cdn.yoda.slideslive.com/%s/master.%s' - # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol formats.extend(self._extract_m3u8_formats( - _MANIFEST_PATTERN % (service_id, 'm3u8'), - service_id, 'mp4', m3u8_id='hls', fatal=False)) + f'https://{cdn_hostname}/{service_id}/master.m3u8', + video_id, 'mp4', m3u8_id='hls', fatal=False, live=True)) formats.extend(self._extract_mpd_formats( - _MANIFEST_PATTERN % (service_id, 'mpd'), service_id, - mpd_id='dash', fatal=False)) + f'https://{cdn_hostname}/{service_id}/master.mpd', + video_id, mpd_id='dash', fatal=False)) info.update({ - 'id': service_id, 'formats': formats, }) else: @@ -94,10 +178,11 @@ class SlidesLiveIE(InfoExtractor): '_type': 'url_transparent', 'url': service_id, 'ie_key': service_name.capitalize(), - 'title': video_data.get('title'), + 'display_id': video_id, }) if service_name == 'vimeo': info['url'] = smuggle_url( - 'https://player.vimeo.com/video/' + service_id, + f'https://player.vimeo.com/video/{service_id}', {'http_headers': {'Referer': url}}) + return info -- cgit v1.2.3 From 3d667e0047915c32f5df9fdd86a4223dc0e9ce8f Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Thu, 29 Dec 2022 12:03:03 +0000 Subject: [extractor/slideslive] Support embeds and slides (#5784) Authored by: bashonly, Grub4K, pukkandan --- yt_dlp/extractor/slideslive.py | 390 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 362 insertions(+), 28 deletions(-) (limited to 'yt_dlp/extractor/slideslive.py') diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py index 86c26a8a2..4268bfeaf 100644 --- a/yt_dlp/extractor/slideslive.py +++ b/yt_dlp/extractor/slideslive.py @@ -1,16 +1,24 @@ +import re +import urllib.parse + from .common import InfoExtractor from ..utils import ( + ExtractorError, + int_or_none, + parse_qs, smuggle_url, traverse_obj, unified_timestamp, + update_url_query, url_or_none, + xpath_text, ) class SlidesLiveIE(InfoExtractor): - _VALID_URL = r'https?://slideslive\.com/(?P[0-9]+)' + _VALID_URL = r'https?://slideslive\.com/(?:embed/(?:presentation/)?)?(?P[0-9]+)' _TESTS = [{ - # service_name = yoda + # service_name = yoda, only XML slides info 'url': 'https://slideslive.com/38902413/gcc-ia16-backend', 'info_dict': { 'id': '38902413', @@ -19,12 +27,14 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1648189972, 'upload_date': '20220325', 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnails': 'count:42', + 'chapters': 'count:41', }, 'params': { 'skip_download': 'm3u8', }, }, { - # service_name = yoda + # service_name = yoda, /v7/ slides 'url': 'https://slideslive.com/38935785', 'info_dict': { 'id': '38935785', @@ -32,13 +42,15 @@ class SlidesLiveIE(InfoExtractor): 'title': 'Offline Reinforcement Learning: From Algorithms to Practical Challenges', 'upload_date': '20211115', 'timestamp': 1636996003, - 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:640', + 'chapters': 'count:639', }, 'params': { 'skip_download': 'm3u8', }, }, { - # service_name = yoda + # service_name = yoda, /v1/ slides 'url': 'https://slideslive.com/38973182/how-should-a-machine-learning-researcher-think-about-ai-ethics', 'info_dict': { 'id': '38973182', @@ -47,12 +59,14 @@ class SlidesLiveIE(InfoExtractor): 'upload_date': '20220201', 'thumbnail': r're:^https?://.*\.jpg', 'timestamp': 1643728135, + 'thumbnails': 'count:3', + 'chapters': 'count:2', }, 'params': { 'skip_download': 'm3u8', }, }, { - # service_name = youtube + # service_name = youtube, only XML slides info 'url': 'https://slideslive.com/38897546/special-metaprednaska-petra-ludwiga-hodnoty-pro-lepsi-spolecnost', 'md5': '8a79b5e3d700837f40bd2afca3c8fa01', 'info_dict': { @@ -76,26 +90,253 @@ class SlidesLiveIE(InfoExtractor): 'comment_count': int, 'channel_follower_count': int, 'age_limit': 0, - 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnail': r're:^https?://.*\.(?:jpg|webp)', + 'thumbnails': 'count:169', 'playable_in_embed': True, 'availability': 'unlisted', 'tags': [], 'categories': ['People & Blogs'], + 'chapters': 'count:168', + }, + }, { + # embed-only presentation, only XML slides info + 'url': 'https://slideslive.com/embed/presentation/38925850', + 'info_dict': { + 'id': '38925850', + 'ext': 'mp4', + 'title': 'Towards a Deep Network Architecture for Structured Smoothness', + 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnails': 'count:8', + 'timestamp': 1629671508, + 'upload_date': '20210822', + 'chapters': 'count:7', + }, + 'params': { + 'skip_download': 'm3u8', }, }, { - # service_name = youtube + # embed-only presentation, only JSON slides info, /v5/ slides (.png) + 'url': 'https://slideslive.com/38979920/', + 'info_dict': { + 'id': '38979920', + 'ext': 'mp4', + 'title': 'MoReL: Multi-omics Relational Learning', + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:7', + 'timestamp': 1654714970, + 'upload_date': '20220608', + 'chapters': 'count:6', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v2/ slides (.jpg) + 'url': 'https://slideslive.com/38954074', + 'info_dict': { + 'id': '38954074', + 'ext': 'mp4', + 'title': 'Decentralized Attribution of Generative Models', + 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnails': 'count:16', + 'timestamp': 1622806321, + 'upload_date': '20210604', + 'chapters': 'count:15', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v4/ slides (.png) + 'url': 'https://slideslive.com/38979570/', + 'info_dict': { + 'id': '38979570', + 'ext': 'mp4', + 'title': 'Efficient Active Search for Combinatorial Optimization Problems', + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:9', + 'timestamp': 1654714896, + 'upload_date': '20220608', + 'chapters': 'count:8', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v10/ slides + 'url': 'https://slideslive.com/embed/presentation/38979880?embed_parent_url=https%3A%2F%2Fedit.videoken.com%2F', + 'info_dict': { + 'id': '38979880', + 'ext': 'mp4', + 'title': 'The Representation Power of Neural Networks', + 'timestamp': 1654714962, + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:22', + 'upload_date': '20220608', + 'chapters': 'count:21', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v7/ slides, 2 video slides + 'url': 'https://slideslive.com/embed/presentation/38979682?embed_container_origin=https%3A%2F%2Fedit.videoken.com', + 'playlist_count': 3, + 'info_dict': { + 'id': '38979682-playlist', + 'title': 'LoRA: Low-Rank Adaptation of Large Language Models', + }, + 'playlist': [{ + 'info_dict': { + 'id': '38979682', + 'ext': 'mp4', + 'title': 'LoRA: Low-Rank Adaptation of Large Language Models', + 'timestamp': 1654714920, + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:30', + 'upload_date': '20220608', + 'chapters': 'count:31', + }, + }, { + 'info_dict': { + 'id': '38979682-021', + 'ext': 'mp4', + 'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 021', + 'duration': 3, + 'timestamp': 1654714920, + 'upload_date': '20220608', + }, + }, { + 'info_dict': { + 'id': '38979682-024', + 'ext': 'mp4', + 'title': 'LoRA: Low-Rank Adaptation of Large Language Models - Slide 024', + 'duration': 4, + 'timestamp': 1654714920, + 'upload_date': '20220608', + }, + }], + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v6/ slides, 1 video slide, edit.videoken.com embed + 'url': 'https://slideslive.com/38979481/', + 'playlist_count': 2, + 'info_dict': { + 'id': '38979481-playlist', + 'title': 'How to Train Your MAML to Excel in Few-Shot Classification', + }, + 'playlist': [{ + 'info_dict': { + 'id': '38979481', + 'ext': 'mp4', + 'title': 'How to Train Your MAML to Excel in Few-Shot Classification', + 'timestamp': 1654714877, + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:43', + 'upload_date': '20220608', + 'chapters': 'count:43', + }, + }, { + 'info_dict': { + 'id': '38979481-013', + 'ext': 'mp4', + 'title': 'How to Train Your MAML to Excel in Few-Shot Classification - Slide 013', + 'duration': 3, + 'timestamp': 1654714877, + 'upload_date': '20220608', + }, + }], + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # /v3/ slides, .jpg and .png, service_name = youtube + 'url': 'https://slideslive.com/embed/38932460/', + 'info_dict': { + 'id': 'RTPdrgkyTiE', + 'display_id': '38932460', + 'ext': 'mp4', + 'title': 'Active Learning for Hierarchical Multi-Label Classification', + 'description': 'Watch full version of this video at https://slideslive.com/38932460.', + 'channel': 'SlidesLive Videos - A', + 'channel_id': 'UC62SdArr41t_-_fX40QCLRw', + 'channel_url': 'https://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw', + 'uploader': 'SlidesLive Videos - A', + 'uploader_id': 'UC62SdArr41t_-_fX40QCLRw', + 'uploader_url': 'http://www.youtube.com/channel/UC62SdArr41t_-_fX40QCLRw', + 'upload_date': '20200903', + 'timestamp': 1602599092, + 'duration': 942, + 'age_limit': 0, + 'live_status': 'not_live', + 'playable_in_embed': True, + 'availability': 'unlisted', + 'categories': ['People & Blogs'], + 'tags': [], + 'channel_follower_count': int, + 'like_count': int, + 'view_count': int, + 'thumbnail': r're:^https?://.*\.(?:jpg|png|webp)', + 'thumbnails': 'count:21', + 'chapters': 'count:20', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + # service_name = yoda 'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend', 'only_matching': True, }, { - # service_name = url + # dead link, service_name = url 'url': 'https://slideslive.com/38922070/learning-transferable-skills-1', 'only_matching': True, }, { - # service_name = vimeo + # dead link, service_name = vimeo 'url': 'https://slideslive.com/38921896/retrospectives-a-venue-for-selfreflection-in-ml-research-3', 'only_matching': True, }] + _WEBPAGE_TESTS = [{ + # only XML slides info + 'url': 'https://iclr.cc/virtual_2020/poster_Hklr204Fvr.html', + 'info_dict': { + 'id': '38925850', + 'ext': 'mp4', + 'title': 'Towards a Deep Network Architecture for Structured Smoothness', + 'thumbnail': r're:^https?://.*\.jpg', + 'thumbnails': 'count:8', + 'timestamp': 1629671508, + 'upload_date': '20210822', + 'chapters': 'count:7', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }] + + @classmethod + def _extract_embed_urls(cls, url, webpage): + # Reference: https://slideslive.com/embed_presentation.js + for embed_id in re.findall(r'(?s)new\s+SlidesLiveEmbed\s*\([^)]+\bpresentationId:\s*["\'](\d+)["\']', webpage): + url_parsed = urllib.parse.urlparse(url) + origin = f'{url_parsed.scheme}://{url_parsed.netloc}' + yield update_url_query( + f'https://slideslive.com/embed/presentation/{embed_id}', { + 'embed_parent_url': url, + 'embed_container_origin': origin, + }) + + def _download_embed_webpage_handle(self, video_id, headers): + return self._download_webpage_handle( + f'https://slideslive.com/embed/presentation/{video_id}', video_id, + headers=headers, query=traverse_obj(headers, { + 'embed_parent_url': 'Referer', + 'embed_container_origin': 'Origin', + })) + def _extract_custom_m3u8_info(self, m3u8_data): m3u8_dict = {} @@ -108,6 +349,8 @@ class SlidesLiveIE(InfoExtractor): 'VOD-VIDEO-ID': 'service_id', 'VOD-VIDEO-SERVERS': 'video_servers', 'VOD-SUBTITLES': 'subtitles', + 'VOD-SLIDES-JSON-URL': 'slides_json_url', + 'VOD-SLIDES-XML-URL': 'slides_xml_url', } for line in m3u8_data.splitlines(): @@ -126,9 +369,33 @@ class SlidesLiveIE(InfoExtractor): return m3u8_dict + def _extract_formats(self, cdn_hostname, path, video_id): + formats = [] + formats.extend(self._extract_m3u8_formats( + f'https://{cdn_hostname}/{path}/master.m3u8', + video_id, 'mp4', m3u8_id='hls', fatal=False, live=True)) + formats.extend(self._extract_mpd_formats( + f'https://{cdn_hostname}/{path}/master.mpd', + video_id, mpd_id='dash', fatal=False)) + return formats + def _real_extract(self, url): video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + webpage, urlh = self._download_embed_webpage_handle( + video_id, headers=traverse_obj(parse_qs(url), { + 'Referer': ('embed_parent_url', -1), + 'Origin': ('embed_container_origin', -1)})) + redirect_url = urlh.geturl() + if 'domain_not_allowed' in redirect_url: + domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False) + if not domain: + raise ExtractorError( + 'This is an embed-only presentation. Try passing --referer', expected=True) + webpage, _ = self._download_embed_webpage_handle(video_id, headers={ + 'Referer': f'https://{domain}/', + 'Origin': f'https://{domain}', + }) + player_token = self._search_regex(r'data-player-token="([^"]+)"', webpage, 'player token') player_data = self._download_webpage( f'https://ben.slideslive.com/player/{video_id}', video_id, @@ -139,6 +406,50 @@ class SlidesLiveIE(InfoExtractor): assert service_name in ('url', 'yoda', 'vimeo', 'youtube') service_id = player_info['service_id'] + slides_info_url = None + slides, slides_info = [], [] + if player_info.get('slides_json_url'): + slides_info_url = player_info['slides_json_url'] + slides = traverse_obj(self._download_json( + slides_info_url, video_id, fatal=False, + note='Downloading slides JSON', errnote=False), 'slides', expected_type=list) or [] + for slide_id, slide in enumerate(slides, start=1): + slides_info.append(( + slide_id, traverse_obj(slide, ('image', 'name')), + int_or_none(slide.get('time'), scale=1000))) + + if not slides and player_info.get('slides_xml_url'): + slides_info_url = player_info['slides_xml_url'] + slides = self._download_xml( + slides_info_url, video_id, fatal=False, + note='Downloading slides XML', errnote='Failed to download slides info') + for slide_id, slide in enumerate(slides.findall('./slide'), start=1): + slides_info.append(( + slide_id, xpath_text(slide, './slideName', 'name'), + int_or_none(xpath_text(slide, './timeSec', 'time')))) + + slides_version = int(self._search_regex( + r'https?://slides\.slideslive\.com/\d+/v(\d+)/\w+\.(?:json|xml)', + slides_info_url, 'slides version', default=0)) + if slides_version < 4: + slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s.jpg' + else: + slide_url_template = 'https://slides.slideslive.com/%s/slides/original/%s.png' + + chapters, thumbnails = [], [] + if url_or_none(player_info.get('thumbnail')): + thumbnails.append({'id': 'cover', 'url': player_info['thumbnail']}) + for slide_id, slide_path, start_time in slides_info: + if slide_path: + thumbnails.append({ + 'id': f'{slide_id:03d}', + 'url': slide_url_template % (video_id, slide_path), + }) + chapters.append({ + 'title': f'Slide {slide_id:03d}', + 'start_time': start_time, + }) + subtitles = {} for sub in traverse_obj(player_info, ('subtitles', ...), expected_type=dict): webvtt_url = url_or_none(sub.get('webvtt_url')) @@ -154,25 +465,15 @@ class SlidesLiveIE(InfoExtractor): 'title': player_info.get('title') or self._html_search_meta('title', webpage, default=''), 'timestamp': unified_timestamp(player_info.get('timestamp')), 'is_live': player_info.get('playlist_type') != 'vod', - 'thumbnail': url_or_none(player_info.get('thumbnail')), + 'thumbnails': thumbnails, + 'chapters': chapters, 'subtitles': subtitles, } - if service_name in ('url', 'yoda'): - if service_name == 'url': - info['url'] = service_id - else: - cdn_hostname = player_info['video_servers'][0] - formats = [] - formats.extend(self._extract_m3u8_formats( - f'https://{cdn_hostname}/{service_id}/master.m3u8', - video_id, 'mp4', m3u8_id='hls', fatal=False, live=True)) - formats.extend(self._extract_mpd_formats( - f'https://{cdn_hostname}/{service_id}/master.mpd', - video_id, mpd_id='dash', fatal=False)) - info.update({ - 'formats': formats, - }) + if service_name == 'url': + info['url'] = service_id + elif service_name == 'yoda': + info['formats'] = self._extract_formats(player_info['video_servers'][0], service_id, video_id) else: info.update({ '_type': 'url_transparent', @@ -185,4 +486,37 @@ class SlidesLiveIE(InfoExtractor): f'https://player.vimeo.com/video/{service_id}', {'http_headers': {'Referer': url}}) - return info + video_slides = traverse_obj(slides, (..., 'video', 'id')) + if not video_slides: + return info + + def entries(): + yield info + + service_data = self._download_json( + f'https://ben.slideslive.com/player/{video_id}/slides_video_service_data', + video_id, fatal=False, query={ + 'player_token': player_token, + 'videos': ','.join(video_slides), + }, note='Downloading video slides info', errnote='Failed to download video slides info') or {} + + for slide_id, slide in enumerate(slides, 1): + if not traverse_obj(slide, ('video', 'service')) == 'yoda': + continue + video_path = traverse_obj(slide, ('video', 'id')) + cdn_hostname = traverse_obj(service_data, ( + video_path, 'video_servers', ...), get_all=False) + if not cdn_hostname or not video_path: + continue + formats = self._extract_formats(cdn_hostname, video_path, video_id) + if not formats: + continue + yield { + 'id': f'{video_id}-{slide_id:03d}', + 'title': f'{info["title"]} - Slide {slide_id:03d}', + 'timestamp': info['timestamp'], + 'duration': int_or_none(traverse_obj(slide, ('video', 'duration_ms')), scale=1000), + 'formats': formats, + } + + return self.playlist_result(entries(), f'{video_id}-playlist', info['title']) -- cgit v1.2.3 From 5ab3534d44231f7711398bc3cfc520e2efd09f50 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Sat, 14 Jan 2023 13:52:03 -0600 Subject: [extractor/slideslive] Fix slides and chapters/duration (#6024) * Fix slides/thumbnails extraction * Extract duration to fix issues w/ `--embed-chapters`, `--split-chapters` * Add `InfoExtractor._extract_mpd_vod_duration` method * Expand applicability of `InfoExtractor._parse_m3u8_vod_duration` method Authored by: bashonly --- yt_dlp/extractor/slideslive.py | 111 +++++++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 33 deletions(-) (limited to 'yt_dlp/extractor/slideslive.py') diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py index 4268bfeaf..3d36edbbc 100644 --- a/yt_dlp/extractor/slideslive.py +++ b/yt_dlp/extractor/slideslive.py @@ -29,6 +29,7 @@ class SlidesLiveIE(InfoExtractor): 'thumbnail': r're:^https?://.*\.jpg', 'thumbnails': 'count:42', 'chapters': 'count:41', + 'duration': 1638, }, 'params': { 'skip_download': 'm3u8', @@ -45,6 +46,7 @@ class SlidesLiveIE(InfoExtractor): 'thumbnail': r're:^https?://.*\.(?:jpg|png)', 'thumbnails': 'count:640', 'chapters': 'count:639', + 'duration': 9832, }, 'params': { 'skip_download': 'm3u8', @@ -61,6 +63,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1643728135, 'thumbnails': 'count:3', 'chapters': 'count:2', + 'duration': 5889, }, 'params': { 'skip_download': 'm3u8', @@ -110,6 +113,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1629671508, 'upload_date': '20210822', 'chapters': 'count:7', + 'duration': 326, }, 'params': { 'skip_download': 'm3u8', @@ -126,6 +130,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1654714970, 'upload_date': '20220608', 'chapters': 'count:6', + 'duration': 171, }, 'params': { 'skip_download': 'm3u8', @@ -142,6 +147,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1622806321, 'upload_date': '20210604', 'chapters': 'count:15', + 'duration': 306, }, 'params': { 'skip_download': 'm3u8', @@ -158,6 +164,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1654714896, 'upload_date': '20220608', 'chapters': 'count:8', + 'duration': 295, }, 'params': { 'skip_download': 'm3u8', @@ -174,6 +181,7 @@ class SlidesLiveIE(InfoExtractor): 'thumbnails': 'count:22', 'upload_date': '20220608', 'chapters': 'count:21', + 'duration': 294, }, 'params': { 'skip_download': 'm3u8', @@ -196,6 +204,7 @@ class SlidesLiveIE(InfoExtractor): 'thumbnails': 'count:30', 'upload_date': '20220608', 'chapters': 'count:31', + 'duration': 272, }, }, { 'info_dict': { @@ -237,6 +246,7 @@ class SlidesLiveIE(InfoExtractor): 'thumbnails': 'count:43', 'upload_date': '20220608', 'chapters': 'count:43', + 'duration': 315, }, }, { 'info_dict': { @@ -285,6 +295,23 @@ class SlidesLiveIE(InfoExtractor): 'params': { 'skip_download': 'm3u8', }, + }, { + # /v3/ slides, .png only, service_name = yoda + 'url': 'https://slideslive.com/38983994', + 'info_dict': { + 'id': '38983994', + 'ext': 'mp4', + 'title': 'Zero-Shot AutoML with Pretrained Models', + 'timestamp': 1662384834, + 'upload_date': '20220905', + 'thumbnail': r're:^https?://.*\.(?:jpg|png)', + 'thumbnails': 'count:23', + 'chapters': 'count:22', + 'duration': 295, + }, + 'params': { + 'skip_download': 'm3u8', + }, }, { # service_name = yoda 'url': 'https://slideslive.com/38903721/magic-a-scientific-resurrection-of-an-esoteric-legend', @@ -311,6 +338,7 @@ class SlidesLiveIE(InfoExtractor): 'timestamp': 1629671508, 'upload_date': '20210822', 'chapters': 'count:7', + 'duration': 326, }, 'params': { 'skip_download': 'm3u8', @@ -369,15 +397,28 @@ class SlidesLiveIE(InfoExtractor): return m3u8_dict - def _extract_formats(self, cdn_hostname, path, video_id): - formats = [] - formats.extend(self._extract_m3u8_formats( + def _extract_formats_and_duration(self, cdn_hostname, path, video_id, skip_duration=False): + formats, duration = [], None + + hls_formats = self._extract_m3u8_formats( f'https://{cdn_hostname}/{path}/master.m3u8', - video_id, 'mp4', m3u8_id='hls', fatal=False, live=True)) - formats.extend(self._extract_mpd_formats( - f'https://{cdn_hostname}/{path}/master.mpd', - video_id, mpd_id='dash', fatal=False)) - return formats + video_id, 'mp4', m3u8_id='hls', fatal=False, live=True) + if hls_formats: + if not skip_duration: + duration = self._extract_m3u8_vod_duration( + hls_formats[0]['url'], video_id, note='Extracting duration from HLS manifest') + formats.extend(hls_formats) + + dash_formats = self._extract_mpd_formats( + f'https://{cdn_hostname}/{path}/master.mpd', video_id, mpd_id='dash', fatal=False) + if dash_formats: + if not duration and not skip_duration: + duration = self._extract_mpd_vod_duration( + f'https://{cdn_hostname}/{path}/master.mpd', video_id, + note='Extracting duration from DASH manifest') + formats.extend(dash_formats) + + return formats, duration def _real_extract(self, url): video_id = self._match_id(url) @@ -406,44 +447,42 @@ class SlidesLiveIE(InfoExtractor): assert service_name in ('url', 'yoda', 'vimeo', 'youtube') service_id = player_info['service_id'] - slides_info_url = None - slides, slides_info = [], [] + slide_url_template = 'https://slides.slideslive.com/%s/slides/original/%s%s' + slides, slides_info = {}, [] + if player_info.get('slides_json_url'): - slides_info_url = player_info['slides_json_url'] - slides = traverse_obj(self._download_json( - slides_info_url, video_id, fatal=False, - note='Downloading slides JSON', errnote=False), 'slides', expected_type=list) or [] - for slide_id, slide in enumerate(slides, start=1): + slides = self._download_json( + player_info['slides_json_url'], video_id, fatal=False, + note='Downloading slides JSON', errnote=False) or {} + slide_ext_default = '.png' + slide_quality = traverse_obj(slides, ('slide_qualities', 0)) + if slide_quality: + slide_ext_default = '.jpg' + slide_url_template = f'https://cdn.slideslive.com/data/presentations/%s/slides/{slide_quality}/%s%s' + for slide_id, slide in enumerate(traverse_obj(slides, ('slides', ...), expected_type=dict), 1): slides_info.append(( slide_id, traverse_obj(slide, ('image', 'name')), + traverse_obj(slide, ('image', 'extname'), default=slide_ext_default), int_or_none(slide.get('time'), scale=1000))) if not slides and player_info.get('slides_xml_url'): - slides_info_url = player_info['slides_xml_url'] slides = self._download_xml( - slides_info_url, video_id, fatal=False, + player_info['slides_xml_url'], video_id, fatal=False, note='Downloading slides XML', errnote='Failed to download slides info') - for slide_id, slide in enumerate(slides.findall('./slide'), start=1): + slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s%s' + for slide_id, slide in enumerate(slides.findall('./slide') if slides else [], 1): slides_info.append(( - slide_id, xpath_text(slide, './slideName', 'name'), + slide_id, xpath_text(slide, './slideName', 'name'), '.jpg', int_or_none(xpath_text(slide, './timeSec', 'time')))) - slides_version = int(self._search_regex( - r'https?://slides\.slideslive\.com/\d+/v(\d+)/\w+\.(?:json|xml)', - slides_info_url, 'slides version', default=0)) - if slides_version < 4: - slide_url_template = 'https://cdn.slideslive.com/data/presentations/%s/slides/big/%s.jpg' - else: - slide_url_template = 'https://slides.slideslive.com/%s/slides/original/%s.png' - chapters, thumbnails = [], [] if url_or_none(player_info.get('thumbnail')): thumbnails.append({'id': 'cover', 'url': player_info['thumbnail']}) - for slide_id, slide_path, start_time in slides_info: + for slide_id, slide_path, slide_ext, start_time in slides_info: if slide_path: thumbnails.append({ 'id': f'{slide_id:03d}', - 'url': slide_url_template % (video_id, slide_path), + 'url': slide_url_template % (video_id, slide_path, slide_ext), }) chapters.append({ 'title': f'Slide {slide_id:03d}', @@ -473,7 +512,12 @@ class SlidesLiveIE(InfoExtractor): if service_name == 'url': info['url'] = service_id elif service_name == 'yoda': - info['formats'] = self._extract_formats(player_info['video_servers'][0], service_id, video_id) + formats, duration = self._extract_formats_and_duration( + player_info['video_servers'][0], service_id, video_id) + info.update({ + 'duration': duration, + 'formats': formats, + }) else: info.update({ '_type': 'url_transparent', @@ -486,7 +530,7 @@ class SlidesLiveIE(InfoExtractor): f'https://player.vimeo.com/video/{service_id}', {'http_headers': {'Referer': url}}) - video_slides = traverse_obj(slides, (..., 'video', 'id')) + video_slides = traverse_obj(slides, ('slides', ..., 'video', 'id')) if not video_slides: return info @@ -500,7 +544,7 @@ class SlidesLiveIE(InfoExtractor): 'videos': ','.join(video_slides), }, note='Downloading video slides info', errnote='Failed to download video slides info') or {} - for slide_id, slide in enumerate(slides, 1): + for slide_id, slide in enumerate(traverse_obj(slides, ('slides', ...)), 1): if not traverse_obj(slide, ('video', 'service')) == 'yoda': continue video_path = traverse_obj(slide, ('video', 'id')) @@ -508,7 +552,8 @@ class SlidesLiveIE(InfoExtractor): video_path, 'video_servers', ...), get_all=False) if not cdn_hostname or not video_path: continue - formats = self._extract_formats(cdn_hostname, video_path, video_id) + formats, _ = self._extract_formats_and_duration( + cdn_hostname, video_path, video_id, skip_duration=True) if not formats: continue yield { -- cgit v1.2.3 From 3d2623a898196640f7cc0fc8b70118ff19e6925d Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 9 Jul 2023 13:23:02 +0530 Subject: [compat, networking] Deprecate old functions (#2861) Authored by: coletdjnz, pukkandan --- yt_dlp/extractor/slideslive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/extractor/slideslive.py') diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py index 3d36edbbc..25f867a60 100644 --- a/yt_dlp/extractor/slideslive.py +++ b/yt_dlp/extractor/slideslive.py @@ -426,7 +426,7 @@ class SlidesLiveIE(InfoExtractor): video_id, headers=traverse_obj(parse_qs(url), { 'Referer': ('embed_parent_url', -1), 'Origin': ('embed_container_origin', -1)})) - redirect_url = urlh.geturl() + redirect_url = urlh.url if 'domain_not_allowed' in redirect_url: domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False) if not domain: -- cgit v1.2.3