diff options
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 8 | ||||
| -rw-r--r-- | yt_dlp/compat.py | 11 | ||||
| -rw-r--r-- | yt_dlp/extractor/extractors.py | 6 | ||||
| -rw-r--r-- | yt_dlp/extractor/fujitv.py | 21 | ||||
| -rw-r--r-- | yt_dlp/extractor/ruv.py | 88 | ||||
| -rw-r--r-- | yt_dlp/extractor/tiktok.py | 37 | ||||
| -rw-r--r-- | yt_dlp/extractor/youtube.py | 15 |
7 files changed, 171 insertions, 15 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 4438e40e9..1fb551754 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -1470,8 +1470,12 @@ class YoutubeDL(object): self.add_extra_info(ie_result, { 'webpage_url': url, 'original_url': url, - 'webpage_url_basename': url_basename(url), - 'webpage_url_domain': get_domain(url), + }) + webpage_url = ie_result.get('webpage_url') + if webpage_url: + self.add_extra_info(ie_result, { + 'webpage_url_basename': url_basename(webpage_url), + 'webpage_url_domain': get_domain(webpage_url), }) if ie is not None: self.add_extra_info(ie_result, { diff --git a/yt_dlp/compat.py b/yt_dlp/compat.py index b97d4512e..2bc6a6b7f 100644 --- a/yt_dlp/compat.py +++ b/yt_dlp/compat.py @@ -134,6 +134,16 @@ except AttributeError: asyncio.run = compat_asyncio_run +try: # >= 3.7 + asyncio.tasks.all_tasks +except AttributeError: + asyncio.tasks.all_tasks = asyncio.tasks.Task.all_tasks + +try: + import websockets as compat_websockets +except ImportError: + compat_websockets = None + # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # See https://github.com/yt-dlp/yt-dlp/issues/792 # https://docs.python.org/3/library/os.path.html#os.path.expanduser @@ -303,6 +313,7 @@ __all__ = [ 'compat_urllib_response', 'compat_urlparse', 'compat_urlretrieve', + 'compat_websockets', 'compat_xml_parse_error', 'compat_xpath', 'compat_zip', diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index 6af32451f..a9dc2289b 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -1374,7 +1374,10 @@ from .megatvcom import ( ) from .rutv import RUTVIE from .ruutu import RuutuIE -from .ruv import RuvIE +from .ruv import ( + RuvIE, + RuvSpilaIE +) from .safari import ( SafariIE, SafariApiIE, @@ -1613,6 +1616,7 @@ from .tiktok import ( TikTokSoundIE, TikTokEffectIE, TikTokTagIE, + TikTokVMIE, DouyinIE, ) from .tinypic import TinyPicIE diff --git a/yt_dlp/extractor/fujitv.py b/yt_dlp/extractor/fujitv.py index f3cb9a6f4..4fdfe12ab 100644 --- a/yt_dlp/extractor/fujitv.py +++ b/yt_dlp/extractor/fujitv.py @@ -7,6 +7,13 @@ from .common import InfoExtractor class FujiTVFODPlus7IE(InfoExtractor): _VALID_URL = r'https?://fod\.fujitv\.co\.jp/title/(?P<sid>[0-9a-z]{4})/(?P<id>[0-9a-z]+)' _BASE_URL = 'https://i.fod.fujitv.co.jp/' + _BITRATE_MAP = { + 300: (320, 180), + 800: (640, 360), + 1200: (1280, 720), + 2000: (1280, 720), + 4000: (1920, 1080), + } _TESTS = [{ 'url': 'https://fod.fujitv.co.jp/title/5d40/5d40110076', @@ -19,6 +26,17 @@ class FujiTVFODPlus7IE(InfoExtractor): 'description': 'md5:b3f51dbfdda162ac4f789e0ff4d65750', 'thumbnail': 'https://i.fod.fujitv.co.jp/img/program/5d40/episode/5d40110076_a.jpg', }, + }, { + 'url': 'https://fod.fujitv.co.jp/title/5d40/5d40810083', + 'info_dict': { + 'id': '5d40810083', + 'ext': 'mp4', + 'title': '#1324 『まる子とオニの子』の巻／『結成！2月をムダにしない会』の巻', + 'description': 'md5:3972d900b896adc8ab1849e310507efa', + 'series': 'ちびまる子ちゃん', + 'series_id': '5d40', + 'thumbnail': 'https://i.fod.fujitv.co.jp/img/program/5d40/episode/5d40810083_a.jpg'}, + 'skip': 'Video available only in one week' }] def _real_extract(self, url): @@ -36,6 +54,9 @@ class FujiTVFODPlus7IE(InfoExtractor): if not src.get('url'): continue fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'mp4') + for f in fmt: + f.update(dict(zip(('height', 'width'), + self._BITRATE_MAP.get(f.get('tbr'), ())))) formats.extend(fmt) subtitles = self._merge_subtitles(subtitles, subs) self._sort_formats(formats, ['tbr']) diff --git a/yt_dlp/extractor/ruv.py b/yt_dlp/extractor/ruv.py index 8f3cc4095..d806ed068 100644 --- a/yt_dlp/extractor/ruv.py +++ b/yt_dlp/extractor/ruv.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( determine_ext, + parse_duration, + traverse_obj, unified_timestamp, ) @@ -99,3 +101,89 @@ class RuvIE(InfoExtractor): 'timestamp': timestamp, 'formats': formats, } + + +class RuvSpilaIE(InfoExtractor): + IE_NAME = 'ruv.is:spila' + _VALID_URL = r'https?://(?:www\.)?ruv\.is/(?:(?:sjon|ut)varp|(?:krakka|ung)ruv)/spila/.+/(?P<series_id>[0-9]+)/(?P<id>[a-z0-9]+)' + _TESTS = [{ + 'url': 'https://www.ruv.is/sjonvarp/spila/ithrottir/30657/9jcnd4', + 'info_dict': { + 'id': '9jcnd4', + 'ext': 'mp4', + 'title': '01.02.2022', + 'chapters': 'count:4', + 'timestamp': 1643743500, + 'upload_date': '20220201', + 'thumbnail': 'https://d38kdhuogyllre.cloudfront.net/fit-in/1960x/filters:quality(65)/hd_posters/94boog-iti3jg.jpg', + 'description': 'Íþróttafréttir.', + 'age_limit': 0, + }, + }, { + 'url': 'https://www.ruv.is/utvarp/spila/i-ljosi-sogunnar/23795/7hqkre', + 'info_dict': { + 'id': '7hqkre', + 'ext': 'mp3', + 'thumbnail': 'https://d38kdhuogyllre.cloudfront.net/fit-in/1960x/filters:quality(65)/hd_posters/7hqkre-7uepao.jpg', + 'description': 'md5:8d7046549daff35e9a3190dc9901a120', + 'chapters': [], + 'upload_date': '20220204', + 'timestamp': 1643965500, + 'title': 'Nellie Bly II', + 'age_limit': 0, + }, + }, { + 'url': 'https://www.ruv.is/ungruv/spila/ungruv/28046/8beuph', + 'only_matching': True + }, { + 'url': 'https://www.ruv.is/krakkaruv/spila/krakkafrettir/30712/9jbgb0', + 'only_matching': True + }] + + def _real_extract(self, url): + display_id, series_id = self._match_valid_url(url).group('id', 'series_id') + program = self._download_json( + 'https://www.ruv.is/gql/', display_id, query={'query': '''{ + Program(id: %s){ + title image description short_description + episodes(id: {value: "%s"}) { + rating title duration file image firstrun description + clips { + time text + } + subtitles { + name value + } + } + } + }''' % (series_id, display_id)})['data']['Program'] + episode = program['episodes'][0] + + subs = {} + for trk in episode.get('subtitles'): + if trk.get('name') and trk.get('value'): + subs.setdefault(trk['name'], []).append({'url': trk['value'], 'ext': 'vtt'}) + + media_url = episode['file'] + if determine_ext(media_url) == 'm3u8': + formats = self._extract_m3u8_formats(media_url, display_id) + else: + formats = [{'url': media_url}] + + clips = [ + {'start_time': parse_duration(c.get('time')), 'title': c.get('text')} + for c in episode.get('clips') or []] + + return { + 'id': display_id, + 'title': traverse_obj(program, ('episodes', 0, 'title'), 'title'), + 'description': traverse_obj( + program, ('episodes', 0, 'description'), 'description', 'short_description', + expected_type=lambda x: x or None), + 'subtitles': subs, + 'thumbnail': episode.get('image', '').replace('$$IMAGESIZE$$', '1960') or None, + 'timestamp': unified_timestamp(episode.get('firstrun')), + 'formats': formats, + 'age_limit': episode.get('rating'), + 'chapters': clips + } diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py index 4150c3ff3..620973a9f 100644 --- a/yt_dlp/extractor/tiktok.py +++ b/yt_dlp/extractor/tiktok.py @@ -14,6 +14,7 @@ from ..compat import ( ) from ..utils import ( ExtractorError, + HEADRequest, int_or_none, join_nonempty, LazyList, @@ -817,3 +818,39 @@ class DouyinIE(TikTokIE): render_data_json, video_id, transform_source=compat_urllib_parse_unquote) return self._parse_aweme_video_web( traverse_obj(render_data, (..., 'aweme', 'detail'), get_all=False), url) + + +class TikTokVMIE(InfoExtractor): + _VALID_URL = r'https?://(?:vm|vt)\.tiktok\.com/(?P<id>\w+)' + IE_NAME = 'vm.tiktok' + + _TESTS = [{ + 'url': 'https://vm.tiktok.com/ZSe4FqkKd', + 'info_dict': { + 'id': '7023491746608712966', + 'ext': 'mp4', + 'title': 'md5:5607564db90271abbbf8294cca77eddd', + 'description': 'md5:5607564db90271abbbf8294cca77eddd', + 'duration': 11, + 'upload_date': '20211026', + 'uploader_id': '7007385080558846981', + 'creator': 'Memes', + 'artist': 'Memes', + 'track': 'original sound', + 'uploader': 'susmandem', + 'timestamp': 1635284105, + 'thumbnail': r're:https://.+\.webp.*', + 'like_count': int, + 'view_count': int, + 'comment_count': int, + 'repost_count': int, + 'uploader_url': 'https://www.tiktok.com/@MS4wLjABAAAAXcNoOEOxVyBzuII_E--T0MeCrLP0ay1Sm6x_n3dluiWEoWZD0VlQOytwad4W0i0n', + } + }, { + 'url': 'https://vt.tiktok.com/ZSe4FqkKd', + 'only_matching': True, + }] + + def _real_extract(self, url): + return self.url_result(self._request_webpage( + HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).geturl(), TikTokIE) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 5750e75d7..d5f9b6962 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -2245,12 +2245,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): get_all=False, expected_type=compat_str) if not player_url: return - if player_url.startswith('//'): - player_url = 'https:' + player_url - elif not re.match(r'https?://', player_url): - player_url = compat_urlparse.urljoin( - 'https://www.youtube.com', player_url) - return player_url + return urljoin('https://www.youtube.com', player_url) def _download_player_url(self, video_id, fatal=False): res = self._download_webpage( @@ -2399,11 +2394,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): """Turn the encrypted n field into a working signature""" if player_url is None: raise ExtractorError('Cannot decrypt nsig without player_url') - if player_url.startswith('//'): - player_url = 'https:' + player_url - elif not re.match(r'https?://', player_url): - player_url = compat_urlparse.urljoin( - 'https://www.youtube.com', player_url) + player_url = urljoin('https://www.youtube.com', player_url) sig_id = ('nsig_value', s) if sig_id in self._player_cache: @@ -3388,7 +3379,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }) lang_subs.append({ 'ext': fmt, - 'url': update_url_query(base_url, query), + 'url': urljoin('https://www.youtube.com', update_url_query(base_url, query)), 'name': sub_name, }) |