diff options
| -rw-r--r-- | .gitignore | 81 | ||||
| -rw-r--r-- | test/test_utils.py | 22 | ||||
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 24 | ||||
| -rw-r--r-- | yt_dlp/__init__.py | 10 | ||||
| -rw-r--r-- | yt_dlp/extractor/common.py | 7 | ||||
| -rw-r--r-- | yt_dlp/extractor/youtube.py | 209 | ||||
| -rw-r--r-- | yt_dlp/options.py | 6 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/ffmpeg.py | 21 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/modify_chapters.py | 13 | ||||
| -rw-r--r-- | yt_dlp/utils.py | 12 |
10 files changed, 146 insertions, 259 deletions
diff --git a/.gitignore b/.gitignore index efdbc9bf0..e7cca0525 100644 --- a/.gitignore +++ b/.gitignore @@ -41,10 +41,8 @@ cookies *.webp *.annotations.xml *.description - # Allow config/media files in testdata !test/** - # Python *.pyc *.pyo @@ -57,7 +55,6 @@ zip/ tmp/ venv/ completions/ - # Misc *~ *.DS_Store @@ -71,10 +68,11 @@ updates_key.pem *.egg-info .tox *.class - # Generated AUTHORS README.txt +README.md +CONTRIBUTING.md .mailmap *.1 *.bash-completion @@ -82,53 +80,67 @@ README.txt *.exe *.tar.gz *.zsh -*.spec test/testdata/player-*.js - # Binary /ytdlp yt-dlp.zip -*.exe - # Text Editor / IDE .idea *.iml .vscode *.sublime-* - # Lazy extractors */extractor/lazy_extractors.py - # Plugins ytdlp_plugins/extractor/* !ytdlp_plugins/extractor/__init__.py !ytdlp_plugins/extractor/sample.py - # VS Code related files -.vscode - # branding +.readthedocs.yml hypervideo.1 hypervideo.bash-completion hypervideo.fish hypervideo hypervideo.exe hypervideo.tar.gz - +setup.py +tox.in # Ignore nonfree JS or SWF bytecode +.github/ISSUE_TEMPLATE/1_broken_site.md +.github/ISSUE_TEMPLATE/2_site_support_request.md +.github/ISSUE_TEMPLATE/3_site_feature_request.md +.github/ISSUE_TEMPLATE/4_bug_report.md +.github/ISSUE_TEMPLATE/5_feature_request.md +.github/ISSUE_TEMPLATE/6_question.md +.github/ISSUE_TEMPLATE_tmpl/1_broken_site.md +.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.md +.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.md +.github/ISSUE_TEMPLATE_tmpl/4_bug_report.md +.github/ISSUE_TEMPLATE_tmpl/5_feature_request.md +.github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE.md .github/ISSUE_TEMPLATE_tmpl.md -.github/PULL_REQUEST_TEMPLATE.md +.github/FUNDING.yml +.github/banner.svg +.github/workflows/build.yml +.github/workflows/core.yml +.github/workflows/download.yml +.github/workflows/quick-test.yml devscripts/create-github-release.py -devscripts/gh-pages/add-version.py -devscripts/gh-pages/generate-download.py -devscripts/gh-pages/sign-versions.py -devscripts/gh-pages/update-copyright.py -devscripts/gh-pages/update-feed.py -devscripts/gh-pages/update-sites.py +devscripts/gh-pages.unused/add-version.py +devscripts/gh-pages.unused/generate-download.py +devscripts/gh-pages.unused/sign-versions.py +devscripts/gh-pages.unused/update-copyright.py +devscripts/gh-pages.unused/update-feed.py +devscripts/gh-pages.unused/update-sites.py devscripts/make_issue_template.py +devscripts/release.sh devscripts/show-downloads-statistics.py devscripts/wine-py2exe.sh +devscripts/make_readme.py +devscripts/update-formulae.py +devscripts/update-version.py test/swftests/.gitignore test/swftests/ArrayAccess.as test/swftests/ClassCall.as @@ -147,9 +159,36 @@ test/swftests/StaticRetrieval.as test/swftests/StringBasics.as test/swftests/StringCharCodeAt.as test/swftests/StringConversion.as +test/swftests.unused/.gitignore +test/swftests.unused/ArrayAccess.as +test/swftests.unused/ClassCall.as +test/swftests.unused/ClassConstruction.as +test/swftests.unused/ConstArrayAccess.as +test/swftests.unused/ConstantInt.as +test/swftests.unused/DictCall.as +test/swftests.unused/EqualsOperator.as +test/swftests.unused/LocalVars.as +test/swftests.unused/MemberAssignment.as +test/swftests.unused/NeOperator.as +test/swftests.unused/PrivateCall.as +test/swftests.unused/PrivateVoidCall.as +test/swftests.unused/StaticAssignment.as +test/swftests.unused/StaticRetrieval.as +test/swftests.unused/StringBasics.as +test/swftests.unused/StringCharCodeAt.as +test/swftests.unused/StringConversion.as test/test_iqiyi_sdk_interpreter.py +test/test_swfinterp.py.disabled +test/test_unicode_literals.py.disabled +test/test_update.py.disabled +test/test_write_annotations.py.disabled +test/test_iqiyi_sdk_interpreter.py~upstream_master test/test_swfinterp.py test/test_update.py test/versions.json yt_dlp/swfinterp.py yt_dlp/update.py +ytdlp_plugins/extractor/__init__.py +ytdlp_plugins/extractor/sample.py +ytdlp_plugins/postprocessor/__init__.py +ytdlp_plugins/postprocessor/sample.py diff --git a/test/test_utils.py b/test/test_utils.py index 7fc431505..9a5e3f0f0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -848,30 +848,52 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), { 'vcodec': 'avc1.77.30', 'acodec': 'mp4a.40.2', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('mp4a.40.2'), { 'vcodec': 'none', 'acodec': 'mp4a.40.2', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('mp4a.40.5,avc1.42001e'), { 'vcodec': 'avc1.42001e', 'acodec': 'mp4a.40.5', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('avc3.640028'), { 'vcodec': 'avc3.640028', 'acodec': 'none', + 'dynamic_range': None, }) self.assertEqual(parse_codecs(', h264,,newcodec,aac'), { 'vcodec': 'h264', 'acodec': 'aac', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('av01.0.05M.08'), { 'vcodec': 'av01.0.05M.08', 'acodec': 'none', + 'dynamic_range': None, + }) + self.assertEqual(parse_codecs('vp9.2'), { + 'vcodec': 'vp9.2', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('av01.0.12M.10.0.110.09.16.09.0'), { + 'vcodec': 'av01.0.12M.10', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('dvhe'), { + 'vcodec': 'dvhe', + 'acodec': 'none', + 'dynamic_range': 'DV', }) self.assertEqual(parse_codecs('theora, vorbis'), { 'vcodec': 'theora', 'acodec': 'vorbis', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('unknownvcodec, unknownacodec'), { 'vcodec': 'unknownvcodec', diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 613816640..18ab19e09 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -2094,25 +2094,14 @@ class YoutubeDL(object): t.get('url'))) def thumbnail_tester(): - if self.params.get('check_formats'): - test_all = True - to_screen = lambda msg: self.to_screen(f'[info] {msg}') - else: - test_all = False - to_screen = self.write_debug - def test_thumbnail(t): - if not test_all and not t.get('_test_url'): - return True - to_screen('Testing thumbnail %s' % t['id']) + self.to_screen(f'[info] Testing thumbnail {t["id"]}') try: self.urlopen(HEADRequest(t['url'])) except network_exceptions as err: - to_screen('Unable to connect to thumbnail %s URL "%s" - %s. Skipping...' % ( - t['id'], t['url'], error_to_compat_str(err))) + self.to_screen(f'[info] Unable to connect to thumbnail {t["id"]} URL {t["url"]!r} - {err}. Skipping...') return False return True - return test_thumbnail for i, t in enumerate(thumbnails): @@ -2122,7 +2111,7 @@ class YoutubeDL(object): t['resolution'] = '%dx%d' % (t['width'], t['height']) t['url'] = sanitize_url(t['url']) - if self.params.get('check_formats') is not False: + if self.params.get('check_formats'): info_dict['thumbnails'] = LazyList(filter(thumbnail_tester(), thumbnails[::-1])).reverse() else: info_dict['thumbnails'] = thumbnails @@ -2301,6 +2290,8 @@ class YoutubeDL(object): format['protocol'] = determine_protocol(format) if format.get('resolution') is None: format['resolution'] = self.format_resolution(format, default=None) + if format.get('dynamic_range') is None and format.get('vcodec') != 'none': + format['dynamic_range'] = 'SDR' # Add HTTP headers, so that external programs can use them from the # json output full_format_info = info_dict.copy() @@ -3186,6 +3177,7 @@ class YoutubeDL(object): format_field(f, 'ext'), self.format_resolution(f), format_field(f, 'fps', '%d'), + format_field(f, 'dynamic_range', '%s', ignore=(None, 'SDR')).replace('HDR', ''), '|', format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes), format_field(f, 'tbr', '%4dk'), @@ -3203,7 +3195,7 @@ class YoutubeDL(object): format_field(f, 'container', ignore=(None, f.get('ext'))), ))), ] for f in formats if f.get('preference') is None or f['preference'] >= -1000] - header_line = ['ID', 'EXT', 'RESOLUTION', 'FPS', '|', ' FILESIZE', ' TBR', 'PROTO', + header_line = ['ID', 'EXT', 'RESOLUTION', 'FPS', 'HDR', '|', ' FILESIZE', ' TBR', 'PROTO', '|', 'VCODEC', ' VBR', 'ACODEC', ' ABR', ' ASR', 'MORE INFO'] else: table = [ @@ -3356,7 +3348,7 @@ class YoutubeDL(object): def _setup_opener(self): timeout_val = self.params.get('socket_timeout') - self._socket_timeout = 600 if timeout_val is None else float(timeout_val) + self._socket_timeout = 20 if timeout_val is None else float(timeout_val) opts_cookiesfrombrowser = self.params.get('cookiesfrombrowser') opts_cookiefile = self.params.get('cookiefile') diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 512627ebd..d8db5754f 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -29,6 +29,7 @@ from .utils import ( expand_path, match_filter_func, MaxDownloadsReached, + parse_duration, preferredencoding, read_batch_urls, RejectedVideoReached, @@ -487,8 +488,14 @@ def _real_main(argv=None): if opts.allsubtitles and not opts.writeautomaticsub: opts.writesubtitles = True # ModifyChapters must run before FFmpegMetadataPP - remove_chapters_patterns = [] + remove_chapters_patterns, remove_ranges = [], [] for regex in opts.remove_chapters: + if regex.startswith('*'): + dur = list(map(parse_duration, regex[1:].split('-'))) + if len(dur) == 2 and all(t is not None for t in dur): + remove_ranges.append(tuple(dur)) + continue + parser.error(f'invalid --remove-chapters time range {regex!r}. Must be of the form ?start-end') try: remove_chapters_patterns.append(re.compile(regex)) except re.error as err: @@ -498,6 +505,7 @@ def _real_main(argv=None): 'key': 'ModifyChapters', 'remove_chapters_patterns': remove_chapters_patterns, 'remove_sponsor_segments': opts.sponsorblock_remove, + 'remove_ranges': remove_ranges, 'sponsorblock_chapter_title': opts.sponsorblock_chapter_title, 'force_keyframes': opts.force_keyframes_at_cuts }) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index dbe7dfcbf..e00d8c42b 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -147,6 +147,8 @@ class InfoExtractor(object): * width Width of the video, if known * height Height of the video, if known * resolution Textual description of width and height + * dynamic_range The dynamic range of the video. One of: + "SDR" (None), "HDR10", "HDR10+, "HDR12", "HLG, "DV" * tbr Average bitrate of audio and video in KBit/s * abr Average audio bitrate in KBit/s * acodec Name of the audio codec in use @@ -233,7 +235,6 @@ class InfoExtractor(object): * "resolution" (optional, string "{width}x{height}", deprecated) * "filesize" (optional, int) - * "_test_url" (optional, bool) - If true, test the URL thumbnail: Full URL to a video thumbnail image. description: Full video description. uploader: Full name of the video uploader. @@ -1508,7 +1509,7 @@ class InfoExtractor(object): regex = r' *((?P<reverse>\+)?(?P<field>[a-zA-Z0-9_]+)((?P<separator>[~:])(?P<limit>.*?))?)? *$' default = ('hidden', 'aud_or_vid', 'hasvid', 'ie_pref', 'lang', 'quality', - 'res', 'fps', 'codec:vp9.2', 'size', 'br', 'asr', + 'res', 'fps', 'hdr:12', 'codec:vp9.2', 'size', 'br', 'asr', 'proto', 'ext', 'hasaud', 'source', 'format_id') # These must not be aliases ytdl_default = ('hasaud', 'lang', 'quality', 'tbr', 'filesize', 'vbr', 'height', 'width', 'proto', 'vext', 'abr', 'aext', @@ -1519,6 +1520,8 @@ class InfoExtractor(object): 'order': ['av0?1', 'vp0?9.2', 'vp0?9', '[hx]265|he?vc?', '[hx]264|avc', 'vp0?8', 'mp4v|h263', 'theora', '', None, 'none']}, 'acodec': {'type': 'ordered', 'regex': True, 'order': ['opus', 'vorbis', 'aac', 'mp?4a?', 'mp3', 'e?a?c-?3', 'dts', '', None, 'none']}, + 'hdr': {'type': 'ordered', 'regex': True, 'field': 'dynamic_range', + 'order': ['dv', '(hdr)?12', r'(hdr)?10\+', '(hdr)?10', 'hlg', '', 'sdr', None]}, 'proto': {'type': 'ordered', 'regex': True, 'field': 'protocol', 'order': ['(ht|f)tps', '(ht|f)tp$', 'm3u8.+', '.*dash', 'ws|websocket', '', 'mms|rtsp', 'none', 'f4']}, 'vext': {'type': 'ordered', 'field': 'video_ext', diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 892993c9b..aa58a22bf 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -258,28 +258,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor): # If True it will raise an error if no login info is provided _LOGIN_REQUIRED = False - r''' # Unused since login is broken - _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' - _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge' - - _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup' - _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge' - _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}' - ''' - def _login(self): """ Attempt to log in to YouTube. - True is returned if successful or skipped. - False is returned if login failed. - If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised. """ - def warn(message): - self.report_warning(message) - - # username+password login is broken if (self._LOGIN_REQUIRED and self.get_param('cookiefile') is None and self.get_param('cookiesfrombrowser') is None): @@ -287,184 +271,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'Login details are needed to download this content', method='cookies') username, password = self._get_login_info() if username: - warn('Logging in using username and password is broken. %s' % self._LOGIN_HINTS['cookies']) - return - - # Everything below this is broken! - r''' - # No authentication to be performed - if username is None: - if self._LOGIN_REQUIRED and self.get_param('cookiefile') is None: - raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True) - # if self.get_param('cookiefile'): # TODO remove 'and False' later - too many people using outdated cookies and open issues, remind them. - # self.to_screen('[Cookies] Reminder - Make sure to always use up to date cookies!') - return True - - login_page = self._download_webpage( - self._LOGIN_URL, None, - note='Downloading login page', - errnote='unable to fetch login page', fatal=False) - if login_page is False: - return - - login_form = self._hidden_inputs(login_page) - - def req(url, f_req, note, errnote): - data = login_form.copy() - data.update({ - 'pstMsg': 1, - 'checkConnection': 'youtube', - 'checkedDomains': 'youtube', - 'hl': 'en', - 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]', - 'f.req': json.dumps(f_req), - 'flowName': 'GlifWebSignIn', - 'flowEntry': 'ServiceLogin', - # TODO: reverse actual botguard identifier generation algo - 'bgRequest': '["identifier",""]', - }) - return self._download_json( - url, None, note=note, errnote=errnote, - transform_source=lambda s: re.sub(r'^[^[]*', '', s), - fatal=False, - data=urlencode_postdata(data), headers={ - 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', - 'Google-Accounts-XSRF': 1, - }) - - lookup_req = [ - username, - None, [], None, 'US', None, None, 2, False, True, - [ - None, None, - [2, 1, None, 1, - 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', - None, [], 4], - 1, [None, None, []], None, None, None, True - ], - username, - ] - - lookup_results = req( - self._LOOKUP_URL, lookup_req, - 'Looking up account info', 'Unable to look up account info') - - if lookup_results is False: - return False - - user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str) - if not user_hash: - warn('Unable to extract user hash') - return False - - challenge_req = [ - user_hash, - None, 1, None, [1, None, None, None, [password, None, True]], - [ - None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4], - 1, [None, None, []], None, None, None, True - ]] - - challenge_results = req( - self._CHALLENGE_URL, challenge_req, - 'Logging in', 'Unable to log in') - - if challenge_results is False: - return - - login_res = try_get(challenge_results, lambda x: x[0][5], list) - if login_res: - login_msg = try_get(login_res, lambda x: x[5], compat_str) - warn( - 'Unable to login: %s' % 'Invalid password' - if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg) - return False - - res = try_get(challenge_results, lambda x: x[0][-1], list) - if not res: - warn('Unable to extract result entry') - return False - - login_challenge = try_get(res, lambda x: x[0][0], list) - if login_challenge: - challenge_str = try_get(login_challenge, lambda x: x[2], compat_str) - if challenge_str == 'TWO_STEP_VERIFICATION': - # SEND_SUCCESS - TFA code has been successfully sent to phone - # QUOTA_EXCEEDED - reached the limit of TFA codes - status = try_get(login_challenge, lambda x: x[5], compat_str) - if status == 'QUOTA_EXCEEDED': - warn('Exceeded the limit of TFA codes, try later') - return False - - tl = try_get(challenge_results, lambda x: x[1][2], compat_str) - if not tl: - warn('Unable to extract TL') - return False - - tfa_code = self._get_tfa_info('2-step verification code') - - if not tfa_code: - warn( - 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>' - '(Note that only TOTP (Google Authenticator App) codes work at this time.)') - return False - - tfa_code = remove_start(tfa_code, 'G-') - - tfa_req = [ - user_hash, None, 2, None, - [ - 9, None, None, None, None, None, None, None, - [None, tfa_code, True, 2] - ]] - - tfa_results = req( - self._TFA_URL.format(tl), tfa_req, - 'Submitting TFA code', 'Unable to submit TFA code') - - if tfa_results is False: - return False - - tfa_res = try_get(tfa_results, lambda x: x[0][5], list) - if tfa_res: - tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str) - warn( - 'Unable to finish TFA: %s' % 'Invalid TFA code' - if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg) - return False - - check_cookie_url = try_get( - tfa_results, lambda x: x[0][-1][2], compat_str) - else: - CHALLENGES = { - 'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.", - 'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.', - 'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.", - } - challenge = CHALLENGES.get( - challenge_str, - '%s returned error %s.' % (self.IE_NAME, challenge_str)) - warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge) - return False - else: - check_cookie_url = try_get(res, lambda x: x[2], compat_str) - - if not check_cookie_url: - warn('Unable to extract CheckCookie URL') - return False - - check_cookie_results = self._download_webpage( - check_cookie_url, None, 'Checking cookie', fatal=False) - - if check_cookie_results is False: - return False - - if 'https://myaccount.google.com/' not in check_cookie_results: - warn('Unable to log in') - return False - - return True - ''' + self.report_warning(f'Cannot login to YouTube using username and password. {self._LOGIN_HINTS["cookies"]}') def _initialize_consent(self): cookies = self._get_cookies('https://www.youtube.com/') @@ -483,10 +290,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): def _real_initialize(self): self._initialize_consent() - if self._downloader is None: - return - if not self._login(): - return + self._login() _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;' _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;' @@ -2849,7 +2653,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Source is given priority since formats that throttle are given lower source_preference # When throttling issue is fully fixed, remove this - self._sort_formats(formats, ('quality', 'res', 'fps', 'source', 'codec:vp9.2', 'lang')) + self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source', 'codec:vp9.2', 'lang')) keywords = get_first(video_details, 'keywords', expected_type=list) or [] if not keywords and webpage: @@ -2895,21 +2699,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # The best resolution thumbnails sometimes does not appear in the webpage # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340 # List of possible thumbnails - Ref: <https://stackoverflow.com/a/20542029> - hq_thumbnail_names = ['maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3'] - # TODO: Test them also? - For some videos, even these don't exist - guaranteed_thumbnail_names = [ + thumbnail_names = [ + 'maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3', 'hqdefault', 'hq1', 'hq2', 'hq3', '0', 'mqdefault', 'mq1', 'mq2', 'mq3', 'default', '1', '2', '3' ] - thumbnail_names = hq_thumbnail_names + guaranteed_thumbnail_names n_thumbnail_names = len(thumbnail_names) thumbnails.extend({ 'url': 'https://i.ytimg.com/vi{webp}/{video_id}/{name}{live}.{ext}'.format( video_id=video_id, name=name, ext=ext, webp='_webp' if ext == 'webp' else '', live='_live' if is_live else ''), - '_test_url': name in hq_thumbnail_names, } for name in thumbnail_names for ext in ('webp', 'jpg')) for thumb in thumbnails: i = next((i for i, t in enumerate(thumbnail_names) if f'/{video_id}/{t}' in thumb['url']), n_thumbnail_names) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index aa774616c..f45332ee1 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1374,7 +1374,11 @@ def parseOpts(overrideArguments=None): postproc.add_option( '--remove-chapters', metavar='REGEX', dest='remove_chapters', action='append', - help='Remove chapters whose title matches the given regular expression. This option can be used multiple times') + help=( + 'Remove chapters whose title matches the given regular expression. ' + 'Time ranges prefixed by a "*" can also be used in place of chapters to remove the specified range. ' + 'Eg: --remove-chapters "*10:15-15:00" --remove-chapters "intro". ' + 'This option can be used multiple times')) postproc.add_option( '--no-remove-chapters', dest='remove_chapters', action='store_const', const=None, help='Do not remove any chapters from the file (default)') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index e6aa2940a..e5595341d 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -10,7 +10,7 @@ import json from .common import AudioConversionError, PostProcessor -from ..compat import compat_str, compat_numeric_types +from ..compat import compat_str from ..utils import ( dfxp2srt, encodeArgument, @@ -664,15 +664,14 @@ class FFmpegMetadataPP(FFmpegPostProcessor): def _get_metadata_opts(self, info): metadata = {} + meta_prefix = 'meta_' def add(meta_list, info_list=None): - if not meta_list: - return - for info_f in variadic(info_list or meta_list): - if isinstance(info.get(info_f), (compat_str, compat_numeric_types)): - for meta_f in variadic(meta_list): - metadata[meta_f] = info[info_f] - break + value = next(( + str(info[key]) for key in [meta_prefix] + list(variadic(info_list or meta_list)) + if info.get(key) is not None), None) + if value not in ('', None): + metadata.update({meta_f: value for meta_f in variadic(meta_list)}) # See [1-4] for some info on media metadata/metadata supported # by ffmpeg. @@ -695,9 +694,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('episode_id', ('episode', 'episode_id')) add('episode_sort', 'episode_number') - prefix = 'meta_' - for key in filter(lambda k: k.startswith(prefix), info.keys()): - add(key[len(prefix):], key) + for key, value in info.items(): + if value is not None and key != meta_prefix and key.startswith(meta_prefix): + metadata[key[len(meta_prefix):]] = value for name, value in metadata.items(): yield ('-metadata', f'{name}={value}') diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py index 72a705fc5..a0818c41b 100644 --- a/yt_dlp/postprocessor/modify_chapters.py +++ b/yt_dlp/postprocessor/modify_chapters.py @@ -20,11 +20,12 @@ DEFAULT_SPONSORBLOCK_CHAPTER_TITLE = '[SponsorBlock]: %(category_names)l' class ModifyChaptersPP(FFmpegPostProcessor): - def __init__(self, downloader, remove_chapters_patterns=None, remove_sponsor_segments=None, - sponsorblock_chapter_title=DEFAULT_SPONSORBLOCK_CHAPTER_TITLE, force_keyframes=False): + def __init__(self, downloader, remove_chapters_patterns=None, remove_sponsor_segments=None, remove_ranges=None, + *, sponsorblock_chapter_title=DEFAULT_SPONSORBLOCK_CHAPTER_TITLE, force_keyframes=False): FFmpegPostProcessor.__init__(self, downloader) self._remove_chapters_patterns = set(remove_chapters_patterns or []) self._remove_sponsor_segments = set(remove_sponsor_segments or []) + self._ranges_to_remove = set(remove_ranges or []) self._sponsorblock_chapter_title = sponsorblock_chapter_title self._force_keyframes = force_keyframes @@ -97,6 +98,14 @@ class ModifyChaptersPP(FFmpegPostProcessor): if warn_no_chapter_to_remove: self.to_screen('There are no matching SponsorBlock chapters') + sponsor_chapters.extend({ + 'start_time': start, + 'end_time': end, + 'category': 'manually_removed', + '_categories': [('manually_removed', start, end)], + 'remove': True, + } for start, end in self._ranges_to_remove) + return chapters, sponsor_chapters def _get_supported_subs(self, info): diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 1c213aa44..c42817e75 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4618,12 +4618,21 @@ def parse_codecs(codecs_str): return {} split_codecs = list(filter(None, map( str.strip, codecs_str.strip().strip(',').split(',')))) - vcodec, acodec = None, None + vcodec, acodec, hdr = None, None, None for full_codec in split_codecs: codec = full_codec.split('.')[0] if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora', 'dvh1', 'dvhe'): if not vcodec: vcodec = full_codec + if codec in ('dvh1', 'dvhe'): + hdr = 'DV' + elif codec == 'vp9' and vcodec.startswith('vp9.2'): + hdr = 'HDR10' + elif codec == 'av01': + parts = full_codec.split('.') + if len(parts) > 3 and parts[3] == '10': + hdr = 'HDR10' + vcodec = '.'.join(parts[:4]) elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'): if not acodec: acodec = full_codec @@ -4639,6 +4648,7 @@ def parse_codecs(codecs_str): return { 'vcodec': vcodec or 'none', 'acodec': acodec or 'none', + 'dynamic_range': hdr, } return {} |