diff options
Diffstat (limited to 'yt_dlp/postprocessor')
| -rw-r--r-- | yt_dlp/postprocessor/common.py | 37 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/embedthumbnail.py | 5 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/ffmpeg.py | 69 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/metadataparser.py | 6 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/modify_chapters.py | 21 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/movefilesafterdownload.py | 2 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/sponsorblock.py | 19 |
7 files changed, 91 insertions, 68 deletions
diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py index 7c63fe8a4..537792b07 100644 --- a/yt_dlp/postprocessor/common.py +++ b/yt_dlp/postprocessor/common.py @@ -1,17 +1,16 @@ import functools -import itertools import json import os -import time import urllib.error from ..utils import ( PostProcessingError, + RetryManager, _configuration_args, + deprecation_warning, encodeFilename, network_exceptions, sanitized_Request, - write_string, ) @@ -74,10 +73,14 @@ class PostProcessor(metaclass=PostProcessorMetaClass): if self._downloader: return self._downloader.report_warning(text, *args, **kwargs) - def deprecation_warning(self, text): + def deprecation_warning(self, msg): + warn = getattr(self._downloader, 'deprecation_warning', deprecation_warning) + return warn(msg, stacklevel=1) + + def deprecated_feature(self, msg): if self._downloader: - return self._downloader.deprecation_warning(text) - write_string(f'DeprecationWarning: {text}') + return self._downloader.deprecated_feature(msg) + return deprecation_warning(msg, stacklevel=1) def report_error(self, text, *args, **kwargs): self.deprecation_warning('"yt_dlp.postprocessor.PostProcessor.report_error" is deprecated. ' @@ -190,27 +193,23 @@ class PostProcessor(metaclass=PostProcessorMetaClass): progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s', progress_dict)) - def _download_json(self, url, *, expected_http_errors=(404,)): + def _retry_download(self, err, count, retries): # While this is not an extractor, it behaves similar to one and - # so obey extractor_retries and sleep_interval_requests - max_retries = self.get_param('extractor_retries', 3) - sleep_interval = self.get_param('sleep_interval_requests') or 0 + # so obey extractor_retries and "--retry-sleep extractor" + RetryManager.report_retry(err, count, retries, info=self.to_screen, warn=self.report_warning, + sleep_func=self.get_param('retry_sleep_functions', {}).get('extractor')) + def _download_json(self, url, *, expected_http_errors=(404,)): self.write_debug(f'{self.PP_NAME} query: {url}') - for retries in itertools.count(): + for retry in RetryManager(self.get_param('extractor_retries', 3), self._retry_download): try: rsp = self._downloader.urlopen(sanitized_Request(url)) - return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8')) except network_exceptions as e: if isinstance(e, urllib.error.HTTPError) and e.code in expected_http_errors: return None - if retries < max_retries: - self.report_warning(f'{e}. Retrying...') - if sleep_interval > 0: - self.to_screen(f'Sleeping {sleep_interval} seconds ...') - time.sleep(sleep_interval) - continue - raise PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}') + retry.error = PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}') + continue + return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8')) class AudioConversionError(PostProcessingError): # Deprecated diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index 606d90d3d..b02d9d499 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -92,7 +92,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor): if info['ext'] == 'mp3': options = [ '-c', 'copy', '-map', '0:0', '-map', '1:0', '-write_id3v1', '1', '-id3v2_version', '3', - '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (front)"'] + '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment=Cover (front)'] self._report_run('ffmpeg', filename) self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options) @@ -139,7 +139,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor): if not success: success = True atomicparsley = next(( - x for x in ['AtomicParsley', 'atomicparsley'] + # libatomicparsley.so : See https://github.com/xibr/ytdlp-lazy/issues/1 + x for x in ['AtomicParsley', 'atomicparsley', 'libatomicparsley.so'] if check_executable(x, ['-v'])), None) if atomicparsley is None: self.to_screen('Neither mutagen nor AtomicParsley was found. Falling back to ffmpeg') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index d0a917379..67890fc31 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1,4 +1,5 @@ import collections +import contextvars import itertools import json import os @@ -9,10 +10,12 @@ import time from .common import PostProcessor from ..compat import functools, imghdr from ..utils import ( + MEDIA_EXTENSIONS, ISO639Utils, Popen, PostProcessingError, _get_exe_version_output, + deprecation_warning, detect_exe_version, determine_ext, dfxp2srt, @@ -28,7 +31,6 @@ from ..utils import ( traverse_obj, variadic, write_json_file, - write_string, ) EXT_TO_OUT_FORMATS = { @@ -81,6 +83,8 @@ class FFmpegPostProcessorError(PostProcessingError): class FFmpegPostProcessor(PostProcessor): + _ffmpeg_location = contextvars.ContextVar('ffmpeg_location', default=None) + def __init__(self, downloader=None): PostProcessor.__init__(self, downloader) self._prefer_ffmpeg = self.get_param('prefer_ffmpeg', True) @@ -100,23 +104,29 @@ class FFmpegPostProcessor(PostProcessor): def _determine_executables(self): programs = [*self._ffmpeg_to_avconv.keys(), *self._ffmpeg_to_avconv.values()] - location = self.get_param('ffmpeg_location') + location = self.get_param('ffmpeg_location', self._ffmpeg_location.get()) if location is None: return {p: p for p in programs} if not os.path.exists(location): - self.report_warning(f'ffmpeg-location {location} does not exist! Continuing without ffmpeg') + self.report_warning( + f'ffmpeg-location {location} does not exist! Continuing without ffmpeg', only_once=True) return {} elif os.path.isdir(location): - dirname, basename = location, None + dirname, basename, filename = location, None, None else: - basename = os.path.splitext(os.path.basename(location))[0] - basename = next((p for p in programs if basename.startswith(p)), 'ffmpeg') + filename = os.path.basename(location) + basename = next((p for p in programs if p in filename), 'ffmpeg') dirname = os.path.dirname(os.path.abspath(location)) if basename in self._ffmpeg_to_avconv.keys(): self._prefer_ffmpeg = True paths = {p: os.path.join(dirname, p) for p in programs} + if basename and basename in filename: + for p in programs: + path = os.path.join(dirname, filename.replace(basename, p)) + if os.path.exists(path): + paths[p] = path if basename: paths[basename] = location return paths @@ -127,7 +137,7 @@ class FFmpegPostProcessor(PostProcessor): path = self._paths.get(prog) if path in self._version_cache: return self._version_cache[path], self._features_cache.get(path, {}) - out = _get_exe_version_output(path, ['-bsfs'], to_screen=self.write_debug) + out = _get_exe_version_output(path, ['-bsfs']) ver = detect_exe_version(out) if out else False if ver: regexs = [ @@ -167,9 +177,9 @@ class FFmpegPostProcessor(PostProcessor): return self.probe_basename def _get_version(self, kind): - executables = (kind, self._ffmpeg_to_avconv[kind]) + executables = (kind, ) if not self._prefer_ffmpeg: - executables = reversed(executables) + executables = (kind, self._ffmpeg_to_avconv[kind]) basename, version, features = next(filter( lambda x: x[1], ((p, *self._get_ffmpeg_version(p)) for p in executables)), (None, None, {})) if kind == 'ffmpeg': @@ -177,8 +187,8 @@ class FFmpegPostProcessor(PostProcessor): else: self.probe_basename = basename if basename == self._ffmpeg_to_avconv[kind]: - self.deprecation_warning( - f'Support for {self._ffmpeg_to_avconv[kind]} is deprecated and may be removed in a future version. Use {kind} instead') + self.deprecated_feature(f'Support for {self._ffmpeg_to_avconv[kind]} is deprecated and ' + f'may be removed in a future version. Use {kind} instead') return version @functools.cached_property @@ -421,7 +431,7 @@ class FFmpegPostProcessor(PostProcessor): class FFmpegExtractAudioPP(FFmpegPostProcessor): - COMMON_AUDIO_EXTS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') + COMMON_AUDIO_EXTS = MEDIA_EXTENSIONS.common_audio + ('wma', ) SUPPORTED_EXTS = tuple(ACODECS.keys()) FORMAT_RE = create_mapping_re(('best', *SUPPORTED_EXTS)) @@ -528,7 +538,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegVideoConvertorPP(FFmpegPostProcessor): - SUPPORTED_EXTS = ('mp4', 'mkv', 'flv', 'webm', 'mov', 'avi', 'mka', 'ogg', *FFmpegExtractAudioPP.SUPPORTED_EXTS) + SUPPORTED_EXTS = (*MEDIA_EXTENSIONS.common_video, *sorted(MEDIA_EXTENSIONS.common_audio + ('aac', 'vorbis'))) FORMAT_RE = create_mapping_re(SUPPORTED_EXTS) _ACTION = 'converting' @@ -586,7 +596,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): filename = info['filepath'] - # Disabled temporarily. There needs to be a way to overide this + # Disabled temporarily. There needs to be a way to override this # in case of duration actually mismatching in extractor # See: https://github.com/yt-dlp/yt-dlp/issues/1870, https://github.com/yt-dlp/yt-dlp/issues/1385 ''' @@ -725,11 +735,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor): value = value.replace('\0', '') # nul character cannot be passed in command line metadata['common'].update({meta_f: value for meta_f in variadic(meta_list)}) - # See [1-4] for some info on media metadata/metadata supported - # by ffmpeg. - # 1. https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/ - # 2. https://wiki.multimedia.cx/index.php/FFmpeg_Metadata - # 3. https://kodi.wiki/view/Video_file_tagging + # Info on media metadata/metadata supported by ffmpeg: + # https://wiki.multimedia.cx/index.php/FFmpeg_Metadata + # https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/ + # https://kodi.wiki/view/Video_file_tagging add('title', ('track', 'title')) add('date', 'upload_date') @@ -798,6 +807,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): class FFmpegMergerPP(FFmpegPostProcessor): + SUPPORTED_EXTS = MEDIA_EXTENSIONS.common_video + @PostProcessor._restrict_to(images=False) def run(self, info): filename = info['filepath'] @@ -922,7 +933,7 @@ class FFmpegFixupDuplicateMoovPP(FFmpegCopyStreamPP): class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): - SUPPORTED_EXTS = ('srt', 'vtt', 'ass', 'lrc') + SUPPORTED_EXTS = MEDIA_EXTENSIONS.subtitles def __init__(self, downloader=None, format=None): super().__init__(downloader) @@ -1044,7 +1055,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): - SUPPORTED_EXTS = ('jpg', 'png', 'webp') + SUPPORTED_EXTS = MEDIA_EXTENSIONS.thumbnails FORMAT_RE = create_mapping_re(SUPPORTED_EXTS) def __init__(self, downloader=None, format=None): @@ -1053,7 +1064,7 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): @classmethod def is_webp(cls, path): - write_string(f'DeprecationWarning: {cls.__module__}.{cls.__name__}.is_webp is deprecated') + deprecation_warning(f'{cls.__module__}.{cls.__name__}.is_webp is deprecated') return imghdr.what(path) == 'webp' def fixup_webp(self, info, idx=-1): @@ -1070,17 +1081,18 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): @staticmethod def _options(target_ext): + yield from ('-update', '1') if target_ext == 'jpg': - return ['-bsf:v', 'mjpeg2jpeg'] - return [] + yield from ('-bsf:v', 'mjpeg2jpeg') def convert_thumbnail(self, thumbnail_filename, target_ext): thumbnail_conv_filename = replace_extension(thumbnail_filename, target_ext) self.to_screen(f'Converting thumbnail "{thumbnail_filename}" to {target_ext}') + _, source_ext = os.path.splitext(thumbnail_filename) self.real_run_ffmpeg( - [(thumbnail_filename, ['-f', 'image2', '-pattern_type', 'none'])], - [(thumbnail_conv_filename.replace('%', '%%'), self._options(target_ext))]) + [(thumbnail_filename, [] if source_ext == '.gif' else ['-f', 'image2', '-pattern_type', 'none'])], + [(thumbnail_conv_filename, self._options(target_ext))]) return thumbnail_conv_filename def run(self, info): @@ -1093,6 +1105,7 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): continue has_thumbnail = True self.fixup_webp(info, idx) + original_thumbnail = thumbnail_dict['filepath'] # Path can change during fixup thumbnail_ext = os.path.splitext(original_thumbnail)[1][1:].lower() if thumbnail_ext == 'jpeg': thumbnail_ext = 'jpg' @@ -1150,9 +1163,9 @@ class FFmpegConcatPP(FFmpegPostProcessor): if len(in_files) < len(entries): raise PostProcessingError('Aborting concatenation because some downloads failed') - ie_copy = self._downloader._playlist_infodict(info) exts = traverse_obj(entries, (..., 'requested_downloads', 0, 'ext'), (..., 'ext')) - ie_copy['ext'] = exts[0] if len(set(exts)) == 1 else 'mkv' + ie_copy = collections.ChainMap({'ext': exts[0] if len(set(exts)) == 1 else 'mkv'}, + info, self._downloader._playlist_infodict(info)) out_file = self._downloader.prepare_filename(ie_copy, 'pl_video') files_to_delete = self.concat_files(in_files, out_file) diff --git a/yt_dlp/postprocessor/metadataparser.py b/yt_dlp/postprocessor/metadataparser.py index 51b927b91..f574f2330 100644 --- a/yt_dlp/postprocessor/metadataparser.py +++ b/yt_dlp/postprocessor/metadataparser.py @@ -1,7 +1,7 @@ import re from .common import PostProcessor -from ..utils import Namespace +from ..utils import Namespace, filter_dict class MetadataParserPP(PostProcessor): @@ -68,9 +68,9 @@ class MetadataParserPP(PostProcessor): if match is None: self.to_screen(f'Could not interpret {inp!r} as {out!r}') return - for attribute, value in match.groupdict().items(): + for attribute, value in filter_dict(match.groupdict()).items(): info[attribute] = value - self.to_screen('Parsed %s from %r: %r' % (attribute, template, value if value is not None else 'NA')) + self.to_screen(f'Parsed {attribute} from {template!r}: {value!r}') template = self.field_to_template(inp) out_re = re.compile(self.format_to_regex(out)) diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py index de3505e11..a745b4524 100644 --- a/yt_dlp/postprocessor/modify_chapters.py +++ b/yt_dlp/postprocessor/modify_chapters.py @@ -16,7 +16,7 @@ class ModifyChaptersPP(FFmpegPostProcessor): *, sponsorblock_chapter_title=DEFAULT_SPONSORBLOCK_CHAPTER_TITLE, force_keyframes=False): FFmpegPostProcessor.__init__(self, downloader) self._remove_chapters_patterns = set(remove_chapters_patterns or []) - self._remove_sponsor_segments = set(remove_sponsor_segments or []) - set(SponsorBlockPP.POI_CATEGORIES.keys()) + self._remove_sponsor_segments = set(remove_sponsor_segments or []) - set(SponsorBlockPP.NON_SKIPPABLE_CATEGORIES.keys()) self._ranges_to_remove = set(remove_ranges or []) self._sponsorblock_chapter_title = sponsorblock_chapter_title self._force_keyframes = force_keyframes @@ -37,9 +37,13 @@ class ModifyChaptersPP(FFmpegPostProcessor): info['chapters'], cuts = self._remove_marked_arrange_sponsors(chapters + sponsor_chapters) if not cuts: return [], info + elif not info['chapters']: + self.report_warning('You have requested to remove the entire video, which is not possible') + return [], info - if self._duration_mismatch(real_duration, info.get('duration'), 1): - if not self._duration_mismatch(real_duration, info['chapters'][-1]['end_time']): + original_duration, info['duration'] = info.get('duration'), info['chapters'][-1]['end_time'] + if self._duration_mismatch(real_duration, original_duration, 1): + if not self._duration_mismatch(real_duration, info['duration']): self.to_screen(f'Skipping {self.pp_key()} since the video appears to be already cut') return [], info if not info.get('__real_download'): @@ -98,7 +102,7 @@ class ModifyChaptersPP(FFmpegPostProcessor): 'start_time': start, 'end_time': end, 'category': 'manually_removed', - '_categories': [('manually_removed', start, end)], + '_categories': [('manually_removed', start, end, 'Manually removed')], 'remove': True, } for start, end in self._ranges_to_remove) @@ -289,13 +293,12 @@ class ModifyChaptersPP(FFmpegPostProcessor): c.pop('_was_cut', None) cats = c.pop('_categories', None) if cats: - category = min(cats, key=lambda c: c[2] - c[1])[0] - cats = orderedSet(x[0] for x in cats) + category, _, _, category_name = min(cats, key=lambda c: c[2] - c[1]) c.update({ 'category': category, - 'categories': cats, - 'name': SponsorBlockPP.CATEGORIES[category], - 'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats] + 'categories': orderedSet(x[0] for x in cats), + 'name': category_name, + 'category_names': orderedSet(x[3] for x in cats), }) c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c.copy()) # Merge identically named sponsors. diff --git a/yt_dlp/postprocessor/movefilesafterdownload.py b/yt_dlp/postprocessor/movefilesafterdownload.py index 436d13227..23b09248c 100644 --- a/yt_dlp/postprocessor/movefilesafterdownload.py +++ b/yt_dlp/postprocessor/movefilesafterdownload.py @@ -1,7 +1,7 @@ import os -import shutil from .common import PostProcessor +from ..compat import shutil from ..utils import ( PostProcessingError, decodeFilename, diff --git a/yt_dlp/postprocessor/sponsorblock.py b/yt_dlp/postprocessor/sponsorblock.py index d79ed7ae7..6ba87cd67 100644 --- a/yt_dlp/postprocessor/sponsorblock.py +++ b/yt_dlp/postprocessor/sponsorblock.py @@ -14,6 +14,10 @@ class SponsorBlockPP(FFmpegPostProcessor): POI_CATEGORIES = { 'poi_highlight': 'Highlight', } + NON_SKIPPABLE_CATEGORIES = { + **POI_CATEGORIES, + 'chapter': 'Chapter', + } CATEGORIES = { 'sponsor': 'Sponsor', 'intro': 'Intermission/Intro Animation', @@ -23,7 +27,7 @@ class SponsorBlockPP(FFmpegPostProcessor): 'filler': 'Filler Tangent', 'interaction': 'Interaction Reminder', 'music_offtopic': 'Non-Music Section', - **POI_CATEGORIES, + **NON_SKIPPABLE_CATEGORIES } def __init__(self, downloader, categories=None, api='https://sponsor.ajay.app'): @@ -60,7 +64,8 @@ class SponsorBlockPP(FFmpegPostProcessor): if duration and duration - start_end[1] <= 1: start_end[1] = duration # SponsorBlock duration may be absent or it may deviate from the real one. - return s['videoDuration'] == 0 or not duration or abs(duration - s['videoDuration']) <= 1 + diff = abs(duration - s['videoDuration']) if s['videoDuration'] else 0 + return diff < 1 or (diff < 5 and diff / (start_end[1] - start_end[0]) < 0.05) duration_match = [s for s in segments if duration_filter(s)] if len(duration_match) != len(segments): @@ -68,17 +73,19 @@ class SponsorBlockPP(FFmpegPostProcessor): def to_chapter(s): (start, end), cat = s['segment'], s['category'] + title = s['description'] if cat == 'chapter' else self.CATEGORIES[cat] return { 'start_time': start, 'end_time': end, 'category': cat, - 'title': self.CATEGORIES[cat], - '_categories': [(cat, start, end)] + 'title': title, + 'type': s['actionType'], + '_categories': [(cat, start, end, title)], } sponsor_chapters = [to_chapter(s) for s in duration_match] if not sponsor_chapters: - self.to_screen('No segments were found in the SponsorBlock database') + self.to_screen('No matching segments were found in the SponsorBlock database') else: self.to_screen(f'Found {len(sponsor_chapters)} segments in the SponsorBlock database') return sponsor_chapters @@ -89,7 +96,7 @@ class SponsorBlockPP(FFmpegPostProcessor): url = f'{self._API_URL}/api/skipSegments/{hash[:4]}?' + urllib.parse.urlencode({ 'service': service, 'categories': json.dumps(self._categories), - 'actionTypes': json.dumps(['skip', 'poi']) + 'actionTypes': json.dumps(['skip', 'poi', 'chapter']) }) for d in self._download_json(url) or []: if d['videoID'] == video_id: |