From 8896899216e90b0ea7ddb1697faf1dc8411b78be Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 2 Jan 2022 03:31:49 +0530 Subject: [FfmpegMetadata] Allow setting metadata of individual streams Closes #877 --- yt_dlp/postprocessor/ffmpeg.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 96b48ded5..97f04d116 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import collections import io import itertools import os @@ -728,15 +729,15 @@ class FFmpegMetadataPP(FFmpegPostProcessor): yield ('-map_metadata', '1') def _get_metadata_opts(self, info): - metadata = {} - meta_prefix = 'meta_' + meta_prefix = 'meta' + metadata = collections.defaultdict(dict) def add(meta_list, info_list=None): value = next(( - str(info[key]) for key in [meta_prefix] + list(variadic(info_list or meta_list)) + str(info[key]) for key in [f'{meta_prefix}_'] + list(variadic(info_list or meta_list)) if info.get(key) is not None), None) if value not in ('', None): - metadata.update({meta_f: value for meta_f in variadic(meta_list)}) + metadata['common'].update({meta_f: value for meta_f in variadic(meta_list)}) # See [1-4] for some info on media metadata/metadata supported # by ffmpeg. @@ -760,22 +761,26 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('episode_sort', 'episode_number') if 'embed-metadata' in self.get_param('compat_opts', []): add('comment', 'description') - metadata.pop('synopsis', None) + metadata['common'].pop('synopsis', None) + meta_regex = rf'{re.escape(meta_prefix)}(?P\d+)?_(?P.+)' for key, value in info.items(): - if value is not None and key != meta_prefix and key.startswith(meta_prefix): - metadata[key[len(meta_prefix):]] = value + mobj = re.fullmatch(meta_regex, key) + if value is not None and mobj: + metadata[mobj.group('i') or 'common'][mobj.group('key')] = value - for name, value in metadata.items(): + for name, value in metadata['common'].items(): yield ('-metadata', f'{name}={value}') stream_idx = 0 for fmt in info.get('requested_formats') or []: stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1 - if fmt.get('language'): - lang = ISO639Utils.short2long(fmt['language']) or fmt['language'] - for i in range(stream_count): - yield ('-metadata:s:%d' % (stream_idx + i), 'language=%s' % lang) + lang = ISO639Utils.short2long(fmt['language']) or fmt.get('language') + for i in range(stream_idx, stream_idx + stream_count): + if lang: + metadata[str(i)].setdefault('language', lang) + for name, value in metadata[str(i)].items(): + yield (f'-metadata:s:{i}', f'{name}={value}') stream_idx += stream_count def _get_infojson_opts(self, info, infofn): -- cgit v1.2.3 From 61e9d9268cad62008f07e635d99cb6ab4120518c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 3 Jan 2022 20:39:46 +0530 Subject: Fix bug in 8896899216e90b0ea7ddb1697faf1dc8411b78be Closes #2215 --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 97f04d116..53e292015 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -775,7 +775,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): stream_idx = 0 for fmt in info.get('requested_formats') or []: stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1 - lang = ISO639Utils.short2long(fmt['language']) or fmt.get('language') + lang = ISO639Utils.short2long(fmt.get('language') or '') or fmt.get('language') for i in range(stream_idx, stream_idx + stream_count): if lang: metadata[str(i)].setdefault('language', lang) -- cgit v1.2.3 From 397235c52bd9dc0e7f993e83b9301d981690c02f Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 12 Jan 2022 08:52:09 +0530 Subject: [ffmpeg] Standardize use of `-map 0` Closes #2182 --- yt_dlp/postprocessor/ffmpeg.py | 52 +++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 24 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 53e292015..7c99fd018 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -13,6 +13,7 @@ from .common import AudioConversionError, PostProcessor from ..compat import compat_str from ..utils import ( + determine_ext, dfxp2srt, encodeArgument, encodeFilename, @@ -191,6 +192,18 @@ class FFmpegPostProcessor(PostProcessor): def probe_executable(self): return self._paths[self.probe_basename] + @staticmethod + def stream_copy_opts(copy=True, *, ext=None): + yield from ('-map', '0') + # Don't copy Apple TV chapters track, bin_data + # See https://github.com/yt-dlp/yt-dlp/issues/2, #19042, #19024, https://trac.ffmpeg.org/ticket/6016 + yield '-dn' + if copy: + yield from ('-c', 'copy') + # For some reason, '-c copy -map 0' is not enough to copy subtitles + if ext in ('mp4', 'mov'): + yield from ('-c:s', 'mov_text') + def get_audio_codec(self, path): if not self.probe_available and not self.available: raise PostProcessingError('ffprobe and ffmpeg not found. Please install or provide the path using --ffmpeg-location') @@ -352,8 +365,9 @@ class FFmpegPostProcessor(PostProcessor): timestamps = timestamps[1:] keyframe_file = prepend_extension(filename, 'keyframes.temp') self.to_screen(f'Re-encoding "{filename}" with appropriate keyframes') - self.run_ffmpeg(filename, keyframe_file, ['-force_key_frames', ','.join( - f'{t:.6f}' for t in timestamps)]) + self.run_ffmpeg(filename, keyframe_file, [ + *self.stream_copy_opts(False, ext=determine_ext(filename)), + '-force_key_frames', ','.join(f'{t:.6f}' for t in timestamps)]) return keyframe_file def concat_files(self, in_files, out_file, concat_opts=None): @@ -368,10 +382,7 @@ class FFmpegPostProcessor(PostProcessor): with open(concat_file, 'wt', encoding='utf-8') as f: f.writelines(self._concat_spec(in_files, concat_opts)) - out_flags = ['-c', 'copy'] - if out_file.rpartition('.')[-1] in ('mp4', 'mov'): - # For some reason, '-c copy' is not enough to copy subtitles - out_flags.extend(['-c:s', 'mov_text']) + out_flags = list(self.stream_copy_opts(ext=determine_ext(out_file))) try: self.real_run_ffmpeg( @@ -574,7 +585,7 @@ class FFmpegVideoRemuxerPP(FFmpegVideoConvertorPP): @staticmethod def _options(target_ext): - return ['-c', 'copy', '-map', '0', '-dn'] + return self.stream_copy_opts() class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): @@ -634,16 +645,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): input_files = [filename] + sub_filenames opts = [ - '-c', 'copy', '-map', '0', '-dn', + *self.stream_copy_opts(ext=info['ext']), # Don't copy the existing subtitles, we may be running the # postprocessor a second time '-map', '-0:s', - # Don't copy Apple TV chapters track, bin_data (see #19042, #19024, - # https://trac.ffmpeg.org/ticket/6016) - '-map', '-0:d', ] - if info['ext'] == 'mp4': - opts += ['-c:s', 'mov_text'] for i, (lang, name) in enumerate(zip(sub_langs, sub_names)): opts.extend(['-map', '%d:0' % (i + 1)]) lang_code = ISO639Utils.short2long(lang) or lang @@ -671,11 +677,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor): @staticmethod def _options(target_ext): - yield from ('-map', '0', '-dn') - if target_ext == 'm4a': + audio_only = target_ext == 'm4a' + yield from self.stream_copy_opts(not audio_only) + if audio_only: yield from ('-vn', '-acodec', 'copy') - else: - yield from ('-c', 'copy') @PostProcessor._restrict_to(images=False) def run(self, info): @@ -859,7 +864,7 @@ class FFmpegFixupStretchedPP(FFmpegFixupPostProcessor): stretched_ratio = info.get('stretched_ratio') if stretched_ratio not in (None, 1): self._fixup('Fixing aspect ratio', info['filepath'], [ - '-c', 'copy', '-map', '0', '-dn', '-aspect', '%f' % stretched_ratio]) + *self.stream_copy_opts(), '-aspect', '%f' % stretched_ratio]) return [], info @@ -867,8 +872,7 @@ class FFmpegFixupM4aPP(FFmpegFixupPostProcessor): @PostProcessor._restrict_to(images=False, video=False) def run(self, info): if info.get('container') == 'm4a_dash': - self._fixup('Correcting container', info['filepath'], [ - '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4']) + self._fixup('Correcting container', info['filepath'], [*self.stream_copy_opts(), '-f', 'mp4']) return [], info @@ -888,7 +892,7 @@ class FFmpegFixupM3u8PP(FFmpegFixupPostProcessor): def run(self, info): if all(self._needs_fixup(info)): self._fixup('Fixing MPEG-TS in MP4 container', info['filepath'], [ - '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']) + *self.stream_copy_opts(), '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']) return [], info @@ -909,7 +913,7 @@ class FFmpegFixupTimestampPP(FFmpegFixupPostProcessor): opts = ['-vf', 'setpts=PTS-STARTPTS'] else: opts = ['-c', 'copy', '-bsf', 'setts=ts=TS-STARTPTS'] - self._fixup('Fixing frame timestamp', info['filepath'], opts + ['-map', '0', '-dn', '-ss', self.trim]) + self._fixup('Fixing frame timestamp', info['filepath'], opts + [*self.stream_copy_opts(False), '-ss', self.trim]) return [], info @@ -918,7 +922,7 @@ class FFmpegCopyStreamPostProcessor(FFmpegFixupPostProcessor): @PostProcessor._restrict_to(images=False) def run(self, info): - self._fixup(self.MESSAGE, info['filepath'], ['-c', 'copy', '-map', '0', '-dn']) + self._fixup(self.MESSAGE, info['filepath'], self.stream_copy_opts()) return [], info @@ -1046,7 +1050,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters)) for idx, chapter in enumerate(chapters): destination, opts = self._ffmpeg_args_for_chapter(idx + 1, chapter, info) - self.real_run_ffmpeg([(in_file, opts)], [(destination, ['-c', 'copy'])]) + self.real_run_ffmpeg([(in_file, opts)], [(destination, self.stream_copy_opts())]) if in_file != info['filepath']: os.remove(in_file) return [], info -- cgit v1.2.3 From ed8d87f911585060faf4df5295fa9ad5bf46c380 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 12 Jan 2022 09:00:21 +0530 Subject: [cleanup, docs] Minor fixes Closes #2230 --- yt_dlp/postprocessor/ffmpeg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 7c99fd018..848fd584a 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -585,7 +585,7 @@ class FFmpegVideoRemuxerPP(FFmpegVideoConvertorPP): @staticmethod def _options(target_ext): - return self.stream_copy_opts() + return FFmpegPostProcessor.stream_copy_opts() class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): @@ -678,7 +678,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): @staticmethod def _options(target_ext): audio_only = target_ext == 'm4a' - yield from self.stream_copy_opts(not audio_only) + yield from FFmpegPostProcessor.stream_copy_opts(not audio_only) if audio_only: yield from ('-vn', '-acodec', 'copy') -- cgit v1.2.3 From 5df1ac92bd85a02696f61a194d9a3a9e1ca34cfc Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 13 Jan 2022 16:09:19 +0530 Subject: [ffmpeg] Ignore unknown streams Closes #2307 --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 848fd584a..43c1b276d 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -197,7 +197,7 @@ class FFmpegPostProcessor(PostProcessor): yield from ('-map', '0') # Don't copy Apple TV chapters track, bin_data # See https://github.com/yt-dlp/yt-dlp/issues/2, #19042, #19024, https://trac.ffmpeg.org/ticket/6016 - yield '-dn' + yield from ('-dn', '-ignore_unknown') if copy: yield from ('-c', 'copy') # For some reason, '-c copy -map 0' is not enough to copy subtitles -- cgit v1.2.3 From 3b603dbdf139efe187f961dbe8b1b24ba16ae194 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 13 Jan 2022 16:31:08 +0530 Subject: Add option `--concat-playlist` Closes #1855, related: #382 --- yt_dlp/postprocessor/ffmpeg.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 43c1b276d..213de0ecf 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1123,3 +1123,48 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): if not has_thumbnail: self.to_screen('There aren\'t any thumbnails to convert') return files_to_delete, info + + +class FFmpegConcatPP(FFmpegPostProcessor): + def __init__(self, downloader, only_multi_video=False): + self._only_multi_video = only_multi_video + super().__init__(downloader) + + def concat_files(self, in_files, out_file): + if len(in_files) == 1: + os.replace(in_files[0], out_file) + return + + codecs = [traverse_obj(self.get_metadata_object(file), ('streams', ..., 'codec_name')) for file in in_files] + if len(set(map(tuple, codecs))) > 1: + raise PostProcessingError( + 'The files have different streams/codecs and cannot be concatenated. ' + 'Either select different formats or --recode-video them to a common format') + super().concat_files(in_files, out_file) + + @PostProcessor._restrict_to(images=False) + def run(self, info): + if not info.get('entries') or self._only_multi_video and info['_type'] != 'multi_video': + return [], info + elif None in info['entries']: + raise PostProcessingError('Aborting concatenation because some downloads failed') + elif any(len(entry) > 1 for entry in traverse_obj(info, ('entries', ..., 'requested_downloads')) or []): + raise PostProcessingError('Concatenation is not supported when downloading multiple separate formats') + + in_files = traverse_obj(info, ('entries', ..., 'requested_downloads', 0, 'filepath')) + if not in_files: + self.to_screen('There are no files to concatenate') + return [], info + + ie_copy = self._downloader._playlist_infodict(info) + exts = [traverse_obj(entry, ('requested_downloads', 0, 'ext'), 'ext') for entry in info['entries']] + ie_copy['ext'] = exts[0] if len(set(exts)) == 1 else 'mkv' + out_file = self._downloader.prepare_filename(ie_copy, 'pl_video') + + self.concat_files(in_files, out_file) + + info['requested_downloads'] = [{ + 'filepath': out_file, + 'ext': ie_copy['ext'], + }] + return in_files, info -- cgit v1.2.3 From 6970b6005e9c07c427d368bbe3f71f85878f325e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 20 Jan 2022 04:27:36 +0530 Subject: [cleanup] Minor fixes Closes #2334 --- yt_dlp/postprocessor/ffmpeg.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 213de0ecf..5b98c7d97 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -568,7 +568,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): else f'already is in target format {source_ext}' if source_ext == target_ext else None) if _skip_msg: - self.to_screen(f'Not {self._ACTION} media file {filename!r}; {_skip_msg}') + self.to_screen(f'Not {self._ACTION} media file "{filename}"; {_skip_msg}') return [], info outpath = replace_extension(filename, target_ext, source_ext) @@ -917,7 +917,7 @@ class FFmpegFixupTimestampPP(FFmpegFixupPostProcessor): return [], info -class FFmpegCopyStreamPostProcessor(FFmpegFixupPostProcessor): +class FFmpegCopyStreamPP(FFmpegFixupPostProcessor): MESSAGE = 'Copying stream' @PostProcessor._restrict_to(images=False) @@ -926,11 +926,11 @@ class FFmpegCopyStreamPostProcessor(FFmpegFixupPostProcessor): return [], info -class FFmpegFixupDurationPP(FFmpegCopyStreamPostProcessor): +class FFmpegFixupDurationPP(FFmpegCopyStreamPP): MESSAGE = 'Fixing video duration' -class FFmpegFixupDuplicateMoovPP(FFmpegCopyStreamPostProcessor): +class FFmpegFixupDuplicateMoovPP(FFmpegCopyStreamPP): MESSAGE = 'Fixing duplicate MOOV atoms' @@ -1132,15 +1132,20 @@ class FFmpegConcatPP(FFmpegPostProcessor): def concat_files(self, in_files, out_file): if len(in_files) == 1: + if os.path.realpath(in_files[0]) != os.path.realpath(out_file): + self.to_screen(f'Moving "{in_files[0]}" to "{out_file}"') os.replace(in_files[0], out_file) - return + return [] codecs = [traverse_obj(self.get_metadata_object(file), ('streams', ..., 'codec_name')) for file in in_files] if len(set(map(tuple, codecs))) > 1: raise PostProcessingError( 'The files have different streams/codecs and cannot be concatenated. ' 'Either select different formats or --recode-video them to a common format') + + self.to_screen(f'Concatenating {len(in_files)} files; Destination: {out_file}') super().concat_files(in_files, out_file) + return in_files @PostProcessor._restrict_to(images=False) def run(self, info): @@ -1161,10 +1166,10 @@ class FFmpegConcatPP(FFmpegPostProcessor): ie_copy['ext'] = exts[0] if len(set(exts)) == 1 else 'mkv' out_file = self._downloader.prepare_filename(ie_copy, 'pl_video') - self.concat_files(in_files, out_file) + files_to_delete = self.concat_files(in_files, out_file) info['requested_downloads'] = [{ 'filepath': out_file, 'ext': ie_copy['ext'], }] - return in_files, info + return files_to_delete, info -- cgit v1.2.3 From 460a1c08b93dec0c95911be2bf2b84a65c3813e8 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 3 Feb 2022 20:26:27 +0530 Subject: [FFmpegConcat] Abort on --skip-download and download errors Closes #2470 --- yt_dlp/postprocessor/ffmpeg.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 5b98c7d97..42e9d12a7 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1149,20 +1149,19 @@ class FFmpegConcatPP(FFmpegPostProcessor): @PostProcessor._restrict_to(images=False) def run(self, info): - if not info.get('entries') or self._only_multi_video and info['_type'] != 'multi_video': + entries = info.get('entries') or [] + if (self.get_param('skip_download') or not any(entries) + or self._only_multi_video and info['_type'] != 'multi_video'): return [], info - elif None in info['entries']: - raise PostProcessingError('Aborting concatenation because some downloads failed') - elif any(len(entry) > 1 for entry in traverse_obj(info, ('entries', ..., 'requested_downloads')) or []): + elif any(len(entry) > 1 for entry in traverse_obj(entries, (..., 'requested_downloads')) or []): raise PostProcessingError('Concatenation is not supported when downloading multiple separate formats') - in_files = traverse_obj(info, ('entries', ..., 'requested_downloads', 0, 'filepath')) - if not in_files: - self.to_screen('There are no files to concatenate') - return [], info + in_files = traverse_obj(entries, (..., 'requested_downloads', 0, 'filepath')) + if len(in_files) < len(entries): + raise PostProcessingError('Aborting concatenation because some downloads failed') ie_copy = self._downloader._playlist_infodict(info) - exts = [traverse_obj(entry, ('requested_downloads', 0, 'ext'), 'ext') for entry in info['entries']] + exts = traverse_obj(entries, (..., 'requested_downloads', 0, 'ext'), (..., 'ext')) ie_copy['ext'] = exts[0] if len(set(exts)) == 1 else 'mkv' out_file = self._downloader.prepare_filename(ie_copy, 'pl_video') -- cgit v1.2.3