From 7a5c1cfe93924351387b44919b3c0b2f66c4b883 Mon Sep 17 00:00:00 2001 From: Pccode66 <49125134+Pccode66@users.noreply.github.com> Date: Wed, 24 Feb 2021 15:45:56 -0300 Subject: Completely change project name to yt-dlp (#85) * All modules and binary names are changed * All documentation references changed * yt-dlp no longer loads youtube-dlc config files * All URLs changed to point to organization account Co-authored-by: Pccode66 Co-authored-by: pukkandan --- yt_dlp/postprocessor/ffmpeg.py | 760 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 760 insertions(+) create mode 100644 yt_dlp/postprocessor/ffmpeg.py (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py new file mode 100644 index 000000000..944a4bc17 --- /dev/null +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -0,0 +1,760 @@ +from __future__ import unicode_literals + +import io +import os +import subprocess +import time +import re +import json + + +from .common import AudioConversionError, PostProcessor + +from ..utils import ( + encodeArgument, + encodeFilename, + get_exe_version, + is_outdated_version, + PostProcessingError, + prepend_extension, + shell_quote, + subtitles_filename, + dfxp2srt, + ISO639Utils, + process_communicate_or_kill, + replace_extension, + traverse_dict, +) + + +EXT_TO_OUT_FORMATS = { + 'aac': 'adts', + 'flac': 'flac', + 'm4a': 'ipod', + 'mka': 'matroska', + 'mkv': 'matroska', + 'mpg': 'mpeg', + 'ogv': 'ogg', + 'ts': 'mpegts', + 'wma': 'asf', + 'wmv': 'asf', +} +ACODECS = { + 'mp3': 'libmp3lame', + 'aac': 'aac', + 'flac': 'flac', + 'm4a': 'aac', + 'opus': 'libopus', + 'vorbis': 'libvorbis', + 'wav': None, +} + + +class FFmpegPostProcessorError(PostProcessingError): + pass + + +class FFmpegPostProcessor(PostProcessor): + def __init__(self, downloader=None): + PostProcessor.__init__(self, downloader) + self._determine_executables() + + def check_version(self): + if not self.available: + raise FFmpegPostProcessorError('ffmpeg not found. Please install') + + required_version = '10-0' if self.basename == 'avconv' else '1.0' + if is_outdated_version( + self._versions[self.basename], required_version): + warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % ( + self.basename, self.basename, required_version) + self.report_warning(warning) + + @staticmethod + def get_versions(downloader=None): + return FFmpegPostProcessor(downloader)._versions + + def _determine_executables(self): + programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] + prefer_ffmpeg = True + + def get_ffmpeg_version(path): + ver = get_exe_version(path, args=['-version']) + if ver: + regexs = [ + r'(?:\d+:)?([0-9.]+)-[0-9]+ubuntu[0-9.]+$', # Ubuntu, see [1] + r'n([0-9.]+)$', # Arch Linux + # 1. http://www.ducea.com/2006/06/17/ubuntu-package-version-naming-explanation/ + ] + for regex in regexs: + mobj = re.match(regex, ver) + if mobj: + ver = mobj.group(1) + return ver + + self.basename = None + self.probe_basename = None + + self._paths = None + self._versions = None + if self._downloader: + prefer_ffmpeg = self.get_param('prefer_ffmpeg', True) + location = self.get_param('ffmpeg_location') + if location is not None: + if not os.path.exists(location): + self.report_warning( + 'ffmpeg-location %s does not exist! ' + 'Continuing without ffmpeg.' % (location)) + self._versions = {} + return + elif not os.path.isdir(location): + basename = os.path.splitext(os.path.basename(location))[0] + if basename not in programs: + self.report_warning( + 'Cannot identify executable %s, its basename should be one of %s. ' + 'Continuing without ffmpeg.' % + (location, ', '.join(programs))) + self._versions = {} + return None + location = os.path.dirname(os.path.abspath(location)) + if basename in ('ffmpeg', 'ffprobe'): + prefer_ffmpeg = True + + self._paths = dict( + (p, os.path.join(location, p)) for p in programs) + self._versions = dict( + (p, get_ffmpeg_version(self._paths[p])) for p in programs) + if self._versions is None: + self._versions = dict( + (p, get_ffmpeg_version(p)) for p in programs) + self._paths = dict((p, p) for p in programs) + + if prefer_ffmpeg is False: + prefs = ('avconv', 'ffmpeg') + else: + prefs = ('ffmpeg', 'avconv') + for p in prefs: + if self._versions[p]: + self.basename = p + break + + if prefer_ffmpeg is False: + prefs = ('avprobe', 'ffprobe') + else: + prefs = ('ffprobe', 'avprobe') + for p in prefs: + if self._versions[p]: + self.probe_basename = p + break + + @property + def available(self): + return self.basename is not None + + @property + def executable(self): + return self._paths[self.basename] + + @property + def probe_available(self): + return self.probe_basename is not None + + @property + def probe_executable(self): + return self._paths[self.probe_basename] + + def get_audio_codec(self, path): + if not self.probe_available and not self.available: + raise PostProcessingError('ffprobe and ffmpeg not found. Please install') + try: + if self.probe_available: + cmd = [ + encodeFilename(self.probe_executable, True), + encodeArgument('-show_streams')] + else: + cmd = [ + encodeFilename(self.executable, True), + encodeArgument('-i')] + cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True)) + self.write_debug('%s command line: %s' % (self.basename, shell_quote(cmd))) + handle = subprocess.Popen( + cmd, stderr=subprocess.PIPE, + stdout=subprocess.PIPE, stdin=subprocess.PIPE) + stdout_data, stderr_data = process_communicate_or_kill(handle) + expected_ret = 0 if self.probe_available else 1 + if handle.wait() != expected_ret: + return None + except (IOError, OSError): + return None + output = (stdout_data if self.probe_available else stderr_data).decode('ascii', 'ignore') + if self.probe_available: + audio_codec = None + for line in output.split('\n'): + if line.startswith('codec_name='): + audio_codec = line.split('=')[1].strip() + elif line.strip() == 'codec_type=audio' and audio_codec is not None: + return audio_codec + else: + # Stream #FILE_INDEX:STREAM_INDEX[STREAM_ID](LANGUAGE): CODEC_TYPE: CODEC_NAME + mobj = re.search( + r'Stream\s*#\d+:\d+(?:\[0x[0-9a-f]+\])?(?:\([a-z]{3}\))?:\s*Audio:\s*([0-9a-z]+)', + output) + if mobj: + return mobj.group(1) + return None + + def get_metadata_object(self, path, opts=[]): + if self.probe_basename != 'ffprobe': + if self.probe_available: + self.report_warning('Only ffprobe is supported for metadata extraction') + raise PostProcessingError('ffprobe not found. Please install.') + self.check_version() + + cmd = [ + encodeFilename(self.probe_executable, True), + encodeArgument('-hide_banner'), + encodeArgument('-show_format'), + encodeArgument('-show_streams'), + encodeArgument('-print_format'), + encodeArgument('json'), + ] + + cmd += opts + cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True)) + self.write_debug('ffprobe command line: %s' % shell_quote(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, stderr = p.communicate() + return json.loads(stdout.decode('utf-8', 'replace')) + + def get_stream_number(self, path, keys, value): + streams = self.get_metadata_object(path)['streams'] + num = next( + (i for i, stream in enumerate(streams) if traverse_dict(stream, keys, casesense=False) == value), + None) + return num, len(streams) + + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + self.check_version() + + oldest_mtime = min( + os.stat(encodeFilename(path)).st_mtime for path in input_paths) + + cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] + # avconv does not have repeat option + if self.basename == 'ffmpeg': + cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] + + def make_args(file, pre=[], post=[], *args, **kwargs): + args = pre + self._configuration_args(*args, **kwargs) + post + return ( + [encodeArgument(o) for o in args] + + [encodeFilename(self._ffmpeg_filename_argument(file), True)]) + + for i, path in enumerate(input_paths): + cmd += make_args(path, post=['-i'], exe='%s_i%d' % (self.basename, i + 1), use_default_arg=False) + cmd += make_args(out_path, pre=opts, exe=self.basename) + + self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, stderr = process_communicate_or_kill(p) + if p.returncode != 0: + stderr = stderr.decode('utf-8', 'replace').strip() + if self.get_param('verbose', False): + self.report_error(stderr) + raise FFmpegPostProcessorError(stderr.split('\n')[-1]) + self.try_utime(out_path, oldest_mtime, oldest_mtime) + return stderr.decode('utf-8', 'replace') + + def run_ffmpeg(self, path, out_path, opts): + return self.run_ffmpeg_multiple_files([path], out_path, opts) + + def _ffmpeg_filename_argument(self, fn): + # Always use 'file:' because the filename may contain ':' (ffmpeg + # interprets that as a protocol) or can start with '-' (-- is broken in + # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details) + # Also leave '-' intact in order not to break streaming to stdout. + if fn.startswith(('http://', 'https://')): + return fn + return 'file:' + fn if fn != '-' else fn + + +class FFmpegExtractAudioPP(FFmpegPostProcessor): + COMMON_AUDIO_EXTENSIONS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') + + def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): + FFmpegPostProcessor.__init__(self, downloader) + if preferredcodec is None: + preferredcodec = 'best' + self._preferredcodec = preferredcodec + self._preferredquality = preferredquality + self._nopostoverwrites = nopostoverwrites + + def run_ffmpeg(self, path, out_path, codec, more_opts): + if codec is None: + acodec_opts = [] + else: + acodec_opts = ['-acodec', codec] + opts = ['-vn'] + acodec_opts + more_opts + try: + FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts) + except FFmpegPostProcessorError as err: + raise AudioConversionError(err.msg) + + def run(self, information): + path = information['filepath'] + orig_ext = information['ext'] + + if self._preferredcodec == 'best' and orig_ext in self.COMMON_AUDIO_EXTENSIONS: + self.to_screen('Skipping audio extraction since the file is already in a common audio format') + return [], information + + filecodec = self.get_audio_codec(path) + if filecodec is None: + raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe') + + more_opts = [] + if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): + if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']: + # Lossless, but in another container + acodec = 'copy' + extension = 'm4a' + more_opts = ['-bsf:a', 'aac_adtstoasc'] + elif filecodec in ['aac', 'flac', 'mp3', 'vorbis', 'opus']: + # Lossless if possible + acodec = 'copy' + extension = filecodec + if filecodec == 'aac': + more_opts = ['-f', 'adts'] + if filecodec == 'vorbis': + extension = 'ogg' + else: + # MP3 otherwise. + acodec = 'libmp3lame' + extension = 'mp3' + more_opts = [] + if self._preferredquality is not None: + if int(self._preferredquality) < 10: + more_opts += ['-q:a', self._preferredquality] + else: + more_opts += ['-b:a', self._preferredquality + 'k'] + else: + # We convert the audio (lossy if codec is lossy) + acodec = ACODECS[self._preferredcodec] + extension = self._preferredcodec + more_opts = [] + if self._preferredquality is not None: + # The opus codec doesn't support the -aq option + if int(self._preferredquality) < 10 and extension != 'opus': + more_opts += ['-q:a', self._preferredquality] + else: + more_opts += ['-b:a', self._preferredquality + 'k'] + if self._preferredcodec == 'aac': + more_opts += ['-f', 'adts'] + if self._preferredcodec == 'm4a': + more_opts += ['-bsf:a', 'aac_adtstoasc'] + if self._preferredcodec == 'vorbis': + extension = 'ogg' + if self._preferredcodec == 'wav': + extension = 'wav' + more_opts += ['-f', 'wav'] + + prefix, sep, ext = path.rpartition('.') # not os.path.splitext, since the latter does not work on unicode in all setups + new_path = prefix + sep + extension + + information['filepath'] = new_path + information['ext'] = extension + + # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly. + if (new_path == path + or (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))): + self.to_screen('Post-process file %s exists, skipping' % new_path) + return [], information + + try: + self.to_screen('Destination: ' + new_path) + self.run_ffmpeg(path, new_path, acodec, more_opts) + except AudioConversionError as e: + raise PostProcessingError( + 'audio conversion failed: ' + e.msg) + except Exception: + raise PostProcessingError('error running ' + self.basename) + + # Try to update the date time for extracted audio file. + if information.get('filetime') is not None: + self.try_utime( + new_path, time.time(), information['filetime'], + errnote='Cannot update utime of audio file') + + return [path], information + + +class FFmpegVideoRemuxerPP(FFmpegPostProcessor): + def __init__(self, downloader=None, preferedformat=None): + super(FFmpegVideoRemuxerPP, self).__init__(downloader) + self._preferedformats = preferedformat.lower().split('/') + + def run(self, information): + path = information['filepath'] + sourceext, targetext = information['ext'].lower(), None + for pair in self._preferedformats: + kv = pair.split('>') + if len(kv) == 1 or kv[0].strip() == sourceext: + targetext = kv[-1].strip() + break + + _skip_msg = ( + 'could not find a mapping for %s' if not targetext + else 'already is in target format %s' if sourceext == targetext + else None) + if _skip_msg: + self.to_screen('Not remuxing media file %s; %s' % (path, _skip_msg % sourceext)) + return [], information + + options = ['-c', 'copy', '-map', '0', '-dn'] + if targetext in ['mp4', 'm4a', 'mov']: + options.extend(['-movflags', '+faststart']) + prefix, sep, oldext = path.rpartition('.') + outpath = prefix + sep + targetext + self.to_screen('Remuxing video from %s to %s; Destination: %s' % (sourceext, targetext, outpath)) + self.run_ffmpeg(path, outpath, options) + information['filepath'] = outpath + information['format'] = targetext + information['ext'] = targetext + return [path], information + + +class FFmpegVideoConvertorPP(FFmpegPostProcessor): + def __init__(self, downloader=None, preferedformat=None): + super(FFmpegVideoConvertorPP, self).__init__(downloader) + self._preferedformat = preferedformat + + def run(self, information): + path = information['filepath'] + if information['ext'] == self._preferedformat: + self.to_screen('Not converting video file %s - already is in target format %s' % (path, self._preferedformat)) + return [], information + options = [] + if self._preferedformat == 'avi': + options.extend(['-c:v', 'libxvid', '-vtag', 'XVID']) + prefix, sep, ext = path.rpartition('.') + outpath = prefix + sep + self._preferedformat + self.to_screen('Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath) + self.run_ffmpeg(path, outpath, options) + information['filepath'] = outpath + information['format'] = self._preferedformat + information['ext'] = self._preferedformat + return [path], information + + +class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): + def __init__(self, downloader=None, already_have_subtitle=False): + super(FFmpegEmbedSubtitlePP, self).__init__(downloader) + self._already_have_subtitle = already_have_subtitle + + def run(self, information): + if information['ext'] not in ('mp4', 'webm', 'mkv'): + self.to_screen('Subtitles can only be embedded in mp4, webm or mkv files') + return [], information + subtitles = information.get('requested_subtitles') + if not subtitles: + self.to_screen('There aren\'t any subtitles to embed') + return [], information + + filename = information['filepath'] + + ext = information['ext'] + sub_langs = [] + sub_filenames = [] + webm_vtt_warn = False + mp4_ass_warn = False + + for lang, sub_info in subtitles.items(): + sub_ext = sub_info['ext'] + if sub_ext == 'json': + self.report_warning('JSON subtitles cannot be embedded') + elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': + sub_langs.append(lang) + sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext)) + else: + if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': + webm_vtt_warn = True + self.report_warning('Only WebVTT subtitles can be embedded in webm files') + if not mp4_ass_warn and ext == 'mp4' and sub_ext == 'ass': + mp4_ass_warn = True + self.report_warning('ASS subtitles cannot be properly embedded in mp4 files; expect issues') + + if not sub_langs: + return [], information + + input_files = [filename] + sub_filenames + + opts = [ + '-c', 'copy', '-map', '0', '-dn', + # Don't copy the existing subtitles, we may be running the + # postprocessor a second time + '-map', '-0:s', + # Don't copy Apple TV chapters track, bin_data (see #19042, #19024, + # https://trac.ffmpeg.org/ticket/6016) + '-map', '-0:d', + ] + if information['ext'] == 'mp4': + opts += ['-c:s', 'mov_text'] + for (i, lang) in enumerate(sub_langs): + opts.extend(['-map', '%d:0' % (i + 1)]) + lang_code = ISO639Utils.short2long(lang) or lang + opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code]) + + temp_filename = prepend_extension(filename, 'temp') + self.to_screen('Embedding subtitles in "%s"' % filename) + self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + files_to_delete = [] if self._already_have_subtitle else sub_filenames + return files_to_delete, information + + +class FFmpegMetadataPP(FFmpegPostProcessor): + def run(self, info): + metadata = {} + + def add(meta_list, info_list=None): + if not info_list: + info_list = meta_list + if not isinstance(meta_list, (list, tuple)): + meta_list = (meta_list,) + if not isinstance(info_list, (list, tuple)): + info_list = (info_list,) + for info_f in info_list: + if info.get(info_f) is not None: + for meta_f in meta_list: + metadata[meta_f] = info[info_f] + break + + # See [1-4] for some info on media metadata/metadata supported + # by ffmpeg. + # 1. https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/ + # 2. https://wiki.multimedia.cx/index.php/FFmpeg_Metadata + # 3. https://kodi.wiki/view/Video_file_tagging + + add('title', ('track', 'title')) + add('date', 'upload_date') + add(('description', 'comment'), 'description') + add('purl', 'webpage_url') + add('track', 'track_number') + add('artist', ('artist', 'creator', 'uploader', 'uploader_id')) + add('genre') + add('album') + add('album_artist') + add('disc', 'disc_number') + add('show', 'series') + add('season_number') + add('episode_id', ('episode', 'episode_id')) + add('episode_sort', 'episode_number') + + if not metadata: + self.to_screen('There isn\'t any metadata to add') + return [], info + + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + in_filenames = [filename] + options = ['-map', '0', '-dn'] + + if info['ext'] == 'm4a': + options.extend(['-vn', '-acodec', 'copy']) + else: + options.extend(['-c', 'copy']) + + for (name, value) in metadata.items(): + options.extend(['-metadata', '%s=%s' % (name, value)]) + + chapters = info.get('chapters', []) + if chapters: + metadata_filename = replace_extension(filename, 'meta') + with io.open(metadata_filename, 'wt', encoding='utf-8') as f: + def ffmpeg_escape(text): + return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text) + + metadata_file_content = ';FFMETADATA1\n' + for chapter in chapters: + metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' + metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000) + metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000) + chapter_title = chapter.get('title') + if chapter_title: + metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) + f.write(metadata_file_content) + in_filenames.append(metadata_filename) + options.extend(['-map_metadata', '1']) + + if '__infojson_filename' in info and info['ext'] in ('mkv', 'mka'): + old_stream, new_stream = self.get_stream_number( + filename, ('tags', 'mimetype'), 'application/json') + if old_stream is not None: + options.extend(['-map', '-0:%d' % old_stream]) + new_stream -= 1 + + options.extend([ + '-attach', info['__infojson_filename'], + '-metadata:s:%d' % new_stream, 'mimetype=application/json' + ]) + + self.to_screen('Adding metadata to \'%s\'' % filename) + self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options) + if chapters: + os.remove(metadata_filename) + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + return [], info + + +class FFmpegMergerPP(FFmpegPostProcessor): + def run(self, info): + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + args = ['-c', 'copy'] + for (i, fmt) in enumerate(info['requested_formats']): + if fmt.get('acodec') != 'none': + args.extend(['-map', '%u:a:0' % (i)]) + if fmt.get('vcodec') != 'none': + args.extend(['-map', '%u:v:0' % (i)]) + self.to_screen('Merging formats into "%s"' % filename) + self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + return info['__files_to_merge'], info + + def can_merge(self): + # TODO: figure out merge-capable ffmpeg version + if self.basename != 'avconv': + return True + + required_version = '10-0' + if is_outdated_version( + self._versions[self.basename], required_version): + warning = ('Your copy of %s is outdated and unable to properly mux separate video and audio files, ' + 'yt-dlp will download single file media. ' + 'Update %s to version %s or newer to fix this.') % ( + self.basename, self.basename, required_version) + self.report_warning(warning) + return False + return True + + +class FFmpegFixupStretchedPP(FFmpegPostProcessor): + def run(self, info): + stretched_ratio = info.get('stretched_ratio') + if stretched_ratio is None or stretched_ratio == 1: + return [], info + + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + + options = ['-c', 'copy', '-map', '0', '-dn', '-aspect', '%f' % stretched_ratio] + self.to_screen('Fixing aspect ratio in "%s"' % filename) + self.run_ffmpeg(filename, temp_filename, options) + + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + return [], info + + +class FFmpegFixupM4aPP(FFmpegPostProcessor): + def run(self, info): + if info.get('container') != 'm4a_dash': + return [], info + + filename = info['filepath'] + temp_filename = prepend_extension(filename, 'temp') + + options = ['-c', 'copy', '-map', '0', '-dn', '-f', 'mp4'] + self.to_screen('Correcting container in "%s"' % filename) + self.run_ffmpeg(filename, temp_filename, options) + + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + return [], info + + +class FFmpegFixupM3u8PP(FFmpegPostProcessor): + def run(self, info): + filename = info['filepath'] + if self.get_audio_codec(filename) == 'aac': + temp_filename = prepend_extension(filename, 'temp') + + options = ['-c', 'copy', '-map', '0', '-dn', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'] + self.to_screen('Fixing malformed AAC bitstream in "%s"' % filename) + self.run_ffmpeg(filename, temp_filename, options) + + os.remove(encodeFilename(filename)) + os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + return [], info + + +class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): + def __init__(self, downloader=None, format=None): + super(FFmpegSubtitlesConvertorPP, self).__init__(downloader) + self.format = format + + def run(self, info): + subs = info.get('requested_subtitles') + filename = info['filepath'] + new_ext = self.format + new_format = new_ext + if new_format == 'vtt': + new_format = 'webvtt' + if subs is None: + self.to_screen('There aren\'t any subtitles to convert') + return [], info + self.to_screen('Converting subtitles') + sub_filenames = [] + for lang, sub in subs.items(): + ext = sub['ext'] + if ext == new_ext: + self.to_screen('Subtitle file for %s is already in the requested format' % new_ext) + continue + elif ext == 'json': + self.to_screen( + 'You have requested to convert json subtitles into another format, ' + 'which is currently not possible') + continue + old_file = subtitles_filename(filename, lang, ext, info.get('ext')) + sub_filenames.append(old_file) + new_file = subtitles_filename(filename, lang, new_ext, info.get('ext')) + + if ext in ('dfxp', 'ttml', 'tt'): + self.report_warning( + 'You have requested to convert dfxp (TTML) subtitles into another format, ' + 'which results in style information loss') + + dfxp_file = old_file + srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext')) + + with open(dfxp_file, 'rb') as f: + srt_data = dfxp2srt(f.read()) + + with io.open(srt_file, 'wt', encoding='utf-8') as f: + f.write(srt_data) + old_file = srt_file + + subs[lang] = { + 'ext': 'srt', + 'data': srt_data + } + + if new_ext == 'srt': + continue + else: + sub_filenames.append(srt_file) + + self.run_ffmpeg(old_file, new_file, ['-f', new_format]) + + with io.open(new_file, 'rt', encoding='utf-8') as f: + subs[lang] = { + 'ext': new_ext, + 'data': f.read(), + } + + return sub_filenames, info -- cgit v1.2.3 From e92caff5d50e60bfd33563d631f0c49ce176dc70 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 9 Mar 2021 07:47:21 +0530 Subject: Refactor (See desc) * Create `FFmpegPostProcessor.real_run_ffmpeg` that can accept multiple input/output files along with switches for each * Rewrite `cli_configuration_args` and related functions * Create `YoutubeDL._ensure_dir_exists` - this was previously defined in multiple places --- yt_dlp/postprocessor/ffmpeg.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 944a4bc17..a8635c1d1 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -234,25 +234,35 @@ class FFmpegPostProcessor(PostProcessor): return num, len(streams) def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + return self.real_run_ffmpeg( + [(path, []) for path in input_paths], + [(out_path, opts)]) + + def real_run_ffmpeg(self, input_path_opts, output_path_opts): self.check_version() oldest_mtime = min( - os.stat(encodeFilename(path)).st_mtime for path in input_paths) + os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts) cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] # avconv does not have repeat option if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] - def make_args(file, pre=[], post=[], *args, **kwargs): - args = pre + self._configuration_args(*args, **kwargs) + post + def make_args(file, args, name, number): + keys = ['_%s%d' % (name, number), '_%s' % name] + if name == 'o' and number == 1: + keys.append('') + args += self._configuration_args(self.basename, keys) + if name == 'i': + args.append('-i') return ( - [encodeArgument(o) for o in args] + [encodeArgument(arg) for arg in args] + [encodeFilename(self._ffmpeg_filename_argument(file), True)]) - for i, path in enumerate(input_paths): - cmd += make_args(path, post=['-i'], exe='%s_i%d' % (self.basename, i + 1), use_default_arg=False) - cmd += make_args(out_path, pre=opts, exe=self.basename) + for arg_type, path_opts in (('i', input_path_opts), ('o', output_path_opts)): + cmd += [arg for i, o in enumerate(path_opts) + for arg in make_args(o[0], o[1], arg_type, i + 1)] self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) @@ -262,7 +272,8 @@ class FFmpegPostProcessor(PostProcessor): if self.get_param('verbose', False): self.report_error(stderr) raise FFmpegPostProcessorError(stderr.split('\n')[-1]) - self.try_utime(out_path, oldest_mtime, oldest_mtime) + for out_path, _ in output_path_opts: + self.try_utime(out_path, oldest_mtime, oldest_mtime) return stderr.decode('utf-8', 'replace') def run_ffmpeg(self, path, out_path, opts): -- cgit v1.2.3 From 7275535116d6b10c1e6df54c50ee872635f3c37e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 15 Mar 2021 04:32:13 +0530 Subject: Split video by chapters (#158) * New options `--split-chapters` and `--no-split-chapters` * The output/path of the split files can be given using the key `chapter` * Additional keys `section_title`, `section_number`, `section_start`, `section_end` are available in the output template * Alias `--split-tracks` for parity with animelover/youtube-dl * `--sponskrub-cut` and `--split-chapter` cannot work together Closes: https://github.com/blackjack4494/yt-dlc/issues/277 https://github.com/ytdl-org/youtube-dl/issues/28438 https://github.com/ytdl-org/youtube-dl/issues/12907 https://github.com/ytdl-org/youtube-dl/issues/6480 https://github.com/ytdl-org/youtube-dl/pull/25005 Rewritten from the implementation by: femaref and Wattux https://github.com/Wattux/youtube-dl/tree/split-at-timestamps https://github.com/ytdl-org/youtube-dl/pull/25005 https://github.com/femaref/youtube-dl/tree/split-track --- yt_dlp/postprocessor/ffmpeg.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index a8635c1d1..7d0452dbc 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -10,6 +10,7 @@ import json from .common import AudioConversionError, PostProcessor +from ..compat import compat_str from ..utils import ( encodeArgument, encodeFilename, @@ -769,3 +770,40 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): } return sub_filenames, info + + +class FFmpegSplitChaptersPP(FFmpegPostProcessor): + + def _prepare_filename(self, number, chapter, info): + info = info.copy() + info.update({ + 'section_number': number, + 'section_title': chapter.get('title'), + 'section_start': chapter.get('start_time'), + 'section_end': chapter.get('end_time'), + }) + return self._downloader.prepare_filename(info, 'chapter') + + def _ffmpeg_args_for_chapter(self, number, chapter, info): + destination = self._prepare_filename(number, chapter, info) + if not self._downloader._ensure_dir_exists(encodeFilename(destination)): + return + + chapter['_filename'] = destination + self.to_screen('Chapter %03d; Destination: %s' % (number, destination)) + return ( + destination, + ['-ss', compat_str(chapter['start_time']), + '-to', compat_str(chapter['end_time'])]) + + def run(self, info): + chapters = info.get('chapters') or [] + if not chapters: + self.report_warning('There are no tracks to extract') + return [], info + + self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters)) + for idx, chapter in enumerate(chapters): + destination, opts = self._ffmpeg_args_for_chapter(idx + 1, chapter, info) + self.real_run_ffmpeg([(info['filepath'], opts)], [(destination, ['-c', 'copy'])]) + return [], info -- cgit v1.2.3 From dcf64d43e0b3a205ebce565d1cf7e71955916824 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 18 Mar 2021 20:54:53 +0530 Subject: [movefiles] Fix bugs and make more robust --- yt_dlp/postprocessor/ffmpeg.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 7d0452dbc..dd07e7c18 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -486,7 +486,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): self.report_warning('JSON subtitles cannot be embedded') elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': sub_langs.append(lang) - sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext)) + sub_filenames.append(sub_info['filepath']) else: if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': webm_vtt_warn = True @@ -732,9 +732,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'You have requested to convert json subtitles into another format, ' 'which is currently not possible') continue - old_file = subtitles_filename(filename, lang, ext, info.get('ext')) + old_file = sub['filepath'] sub_filenames.append(old_file) - new_file = subtitles_filename(filename, lang, new_ext, info.get('ext')) + new_file = replace_extension(old_file, new_ext) if ext in ('dfxp', 'ttml', 'tt'): self.report_warning( @@ -742,7 +742,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'which results in style information loss') dfxp_file = old_file - srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext')) + srt_file = replace_extension(old_file, 'srt') with open(dfxp_file, 'rb') as f: srt_data = dfxp2srt(f.read()) @@ -753,7 +753,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': 'srt', - 'data': srt_data + 'data': srt_data, + 'filepath': srt_file, } if new_ext == 'srt': @@ -767,8 +768,12 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': new_ext, 'data': f.read(), + 'filepath': new_file, } + info['__files_to_move'][new_file] = replace_extension( + info['__files_to_move'][old_file], new_ext) + return sub_filenames, info @@ -789,7 +794,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): if not self._downloader._ensure_dir_exists(encodeFilename(destination)): return - chapter['_filename'] = destination + chapter['filepath'] = destination self.to_screen('Chapter %03d; Destination: %s' % (number, destination)) return ( destination, -- cgit v1.2.3 From a515a78dd3e16b7854844829293ef4e9f3c62319 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 18 Mar 2021 22:18:02 +0530 Subject: fix some typos and linter --- yt_dlp/postprocessor/ffmpeg.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index dd07e7c18..363f10e6d 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -19,7 +19,6 @@ from ..utils import ( PostProcessingError, prepend_extension, shell_quote, - subtitles_filename, dfxp2srt, ISO639Utils, process_communicate_or_kill, @@ -712,7 +711,6 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): def run(self, info): subs = info.get('requested_subtitles') - filename = info['filepath'] new_ext = self.format new_format = new_ext if new_format == 'vtt': -- cgit v1.2.3 From a94bfd6cfe07874f600f0e3c2edd071a1a0a0c4e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 20 Mar 2021 09:15:45 +0530 Subject: [splitchapters] Fix for older ffmpeg Older versions of ffmpeg doesn't allow `-to` as an input option Closes #180 --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 363f10e6d..b0b05f4fc 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -797,7 +797,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): return ( destination, ['-ss', compat_str(chapter['start_time']), - '-to', compat_str(chapter['end_time'])]) + '-t', compat_str(chapter['end_time'] - chapter['start_time'])]) def run(self, info): chapters = info.get('chapters') or [] -- cgit v1.2.3 From cd9b384cc338da6ce43a66ed392ce995ea73eb35 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 19 Mar 2021 17:42:29 +0530 Subject: Embed video URL metadata inside MP4 (#173) `mp4` has `comment` and `synopsis`; the synopsis is expected to have the long description So we save the `webpage_url` to `comment` and `description` to `synopsis` Related: https://github.com/ytdl-org/youtube-dl/issues/28478 Co-authored by: Damiano Amatruda --- yt_dlp/postprocessor/ffmpeg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index b0b05f4fc..ff06a5c31 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -550,8 +550,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('title', ('track', 'title')) add('date', 'upload_date') - add(('description', 'comment'), 'description') - add('purl', 'webpage_url') + add(('description', 'synopsis'), 'description') + add(('purl', 'comment'), 'webpage_url') add('track', 'track_number') add('artist', ('artist', 'creator', 'uploader', 'uploader_id')) add('genre') -- cgit v1.2.3 From beb4b92a66d828816100f34ba77cba76e92e06ea Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 20 Mar 2021 08:50:08 +0530 Subject: More consistent warning messages (#173) Co-authored by: Damiano Amatruda --- yt_dlp/postprocessor/ffmpeg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index ff06a5c31..ffce98cd5 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -61,7 +61,7 @@ class FFmpegPostProcessor(PostProcessor): def check_version(self): if not self.available: - raise FFmpegPostProcessorError('ffmpeg not found. Please install') + raise FFmpegPostProcessorError('ffmpeg not found. Please install or provide the path using --ffmpeg-location') required_version = '10-0' if self.basename == 'avconv' else '1.0' if is_outdated_version( @@ -165,7 +165,7 @@ class FFmpegPostProcessor(PostProcessor): def get_audio_codec(self, path): if not self.probe_available and not self.available: - raise PostProcessingError('ffprobe and ffmpeg not found. Please install') + raise PostProcessingError('ffprobe and ffmpeg not found. Please install or provide the path using --ffmpeg-location') try: if self.probe_available: cmd = [ @@ -207,7 +207,7 @@ class FFmpegPostProcessor(PostProcessor): if self.probe_basename != 'ffprobe': if self.probe_available: self.report_warning('Only ffprobe is supported for metadata extraction') - raise PostProcessingError('ffprobe not found. Please install.') + raise PostProcessingError('ffprobe not found. Please install or provide the path using --ffmpeg-location') self.check_version() cmd = [ @@ -802,7 +802,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): def run(self, info): chapters = info.get('chapters') or [] if not chapters: - self.report_warning('There are no tracks to extract') + self.report_warning('Chapter information is unavailable') return [], info self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters)) -- cgit v1.2.3 From 84601bb72b1ade8867e4a79635823b89ac6754eb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 3 Apr 2021 13:59:55 +0530 Subject: Ability to set a specific field in the file's metadata Eg: `--parse-metadata "description:(?s)(?P.+)"` sets the "comment" field using `description` --- yt_dlp/postprocessor/ffmpeg.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index ffce98cd5..accd715be 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -10,7 +10,7 @@ import json from .common import AudioConversionError, PostProcessor -from ..compat import compat_str +from ..compat import compat_str, compat_numeric_types from ..utils import ( encodeArgument, encodeFilename, @@ -530,6 +530,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): metadata = {} def add(meta_list, info_list=None): + if not meta_list: + return if not info_list: info_list = meta_list if not isinstance(meta_list, (list, tuple)): @@ -537,7 +539,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): if not isinstance(info_list, (list, tuple)): info_list = (info_list,) for info_f in info_list: - if info.get(info_f) is not None: + if isinstance(info.get(info_f), (compat_str, compat_numeric_types)): for meta_f in meta_list: metadata[meta_f] = info[info_f] break @@ -563,6 +565,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('episode_id', ('episode', 'episode_id')) add('episode_sort', 'episode_number') + prefix = 'meta_' + for key in filter(lambda k: k.startswith(prefix), info.keys()): + add(key[len(prefix):], key) + if not metadata: self.to_screen('There isn\'t any metadata to add') return [], info @@ -577,7 +583,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): else: options.extend(['-c', 'copy']) - for (name, value) in metadata.items(): + for name, value in metadata.items(): options.extend(['-metadata', '%s=%s' % (name, value)]) chapters = info.get('chapters', []) -- cgit v1.2.3 From 8fa43c73d83619722c7e30d70247eaa9b7f52810 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 11 Apr 2021 03:48:52 +0530 Subject: Add option `--convert-thumbnails` Closes: https://github.com/yt-dlp/yt-dlp/issues/99 https://github.com/yt-dlp/yt-dlp/issues/102 --- yt_dlp/postprocessor/ffmpeg.py | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index accd715be..0e160f5dc 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -816,3 +816,73 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): destination, opts = self._ffmpeg_args_for_chapter(idx + 1, chapter, info) self.real_run_ffmpeg([(info['filepath'], opts)], [(destination, ['-c', 'copy'])]) return [], info + + +class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): + def __init__(self, downloader=None, format=None): + super(FFmpegThumbnailsConvertorPP, self).__init__(downloader) + self.format = format + + @staticmethod + def is_webp(path): + with open(encodeFilename(path), 'rb') as f: + b = f.read(12) + return b[0:4] == b'RIFF' and b[8:] == b'WEBP' + + def fixup_webp(self, info, idx=-1): + thumbnail_filename = info['thumbnails'][idx]['filepath'] + _, thumbnail_ext = os.path.splitext(thumbnail_filename) + if thumbnail_ext: + thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext != 'webp' and self.is_webp(thumbnail_filename): + self.to_screen('Correcting thumbnail "%s" extension to webp' % thumbnail_filename) + webp_filename = replace_extension(thumbnail_filename, 'webp') + if os.path.exists(webp_filename): + os.remove(webp_filename) + os.rename(encodeFilename(thumbnail_filename), encodeFilename(webp_filename)) + info['thumbnails'][idx]['filepath'] = webp_filename + info['__files_to_move'][webp_filename] = replace_extension( + info['__files_to_move'].pop(thumbnail_filename), 'webp') + + def convert_thumbnail(self, thumbnail_filename, ext): + if ext != 'jpg': + raise FFmpegPostProcessorError('Only conversion to jpg is currently supported') + # NB: % is supposed to be escaped with %% but this does not work + # for input files so working around with standard substitution + escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') + os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) + escaped_thumbnail_jpg_filename = replace_extension(escaped_thumbnail_filename, 'jpg') + self.to_screen('Converting thumbnail "%s" to JPEG' % escaped_thumbnail_filename) + self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_jpg_filename, ['-bsf:v', 'mjpeg2jpeg']) + thumbnail_jpg_filename = replace_extension(thumbnail_filename, 'jpg') + # Rename back to unescaped + os.rename(encodeFilename(escaped_thumbnail_filename), encodeFilename(thumbnail_filename)) + os.rename(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename)) + return thumbnail_jpg_filename + + def run(self, info): + if self.format != 'jpg': + raise FFmpegPostProcessorError('Only conversion to jpg is currently supported') + files_to_delete = [] + has_thumbnail = False + + for idx, thumbnail_dict in enumerate(info['thumbnails']): + if 'filepath' not in thumbnail_dict: + continue + has_thumbnail = True + self.fixup_webp(info, idx) + original_thumbnail = thumbnail_dict['filepath'] + _, thumbnail_ext = os.path.splitext(original_thumbnail) + if thumbnail_ext: + thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext == self.format: + self.to_screen('Thumbnail "%s" is already in the requested format' % original_thumbnail) + continue + thumbnail_dict['filepath'] = self.convert_thumbnail(original_thumbnail, self.format) + files_to_delete.append(original_thumbnail) + info['__files_to_move'][thumbnail_dict['filepath']] = replace_extension( + info['__files_to_move'][original_thumbnail], self.format) + + if not has_thumbnail: + self.to_screen('There aren\'t any thumbnails to convert') + return files_to_delete, info -- cgit v1.2.3 From 41712218233ae144d55f841818df9c63d2bd23d3 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 11 May 2021 14:25:31 +0530 Subject: Add compat-option `no-attach-infojson` --- yt_dlp/postprocessor/ffmpeg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 0e160f5dc..78a80f3f8 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -605,7 +605,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor): in_filenames.append(metadata_filename) options.extend(['-map_metadata', '1']) - if '__infojson_filename' in info and info['ext'] in ('mkv', 'mka'): + if ('no-attach-info-json' not in self.get_param('compat_opts', []) + and '__infojson_filename' in info and info['ext'] in ('mkv', 'mka')): old_stream, new_stream = self.get_stream_number( filename, ('tags', 'mimetype'), 'application/json') if old_stream is not None: -- cgit v1.2.3 From 2412044c90ef3d122eab0b195aaa0f5b2ab27394 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 13 May 2021 01:07:58 +0530 Subject: Add field `name` for subtitles Co-authored by: pukkandan, tpikonen Based on: #310, https://github.com/ytdl-org/youtube-dl/pull/26112 --- yt_dlp/postprocessor/ffmpeg.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 78a80f3f8..b15610829 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -474,8 +474,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): filename = information['filepath'] ext = information['ext'] - sub_langs = [] - sub_filenames = [] + sub_langs, sub_names, sub_filenames = [], [], [] webm_vtt_warn = False mp4_ass_warn = False @@ -485,6 +484,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): self.report_warning('JSON subtitles cannot be embedded') elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': sub_langs.append(lang) + sub_names.append(sub_info.get('name')) sub_filenames.append(sub_info['filepath']) else: if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': @@ -510,10 +510,13 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): ] if information['ext'] == 'mp4': opts += ['-c:s', 'mov_text'] - for (i, lang) in enumerate(sub_langs): + for i, (lang, name) in enumerate(zip(sub_langs, sub_names)): opts.extend(['-map', '%d:0' % (i + 1)]) lang_code = ISO639Utils.short2long(lang) or lang opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code]) + if name: + opts.extend(['-metadata:s:s:%d' % i, 'handler_name=%s' % name, + '-metadata:s:s:%d' % i, 'title=%s' % name]) temp_filename = prepend_extension(filename, 'temp') self.to_screen('Embedding subtitles in "%s"' % filename) -- cgit v1.2.3 From a927acb1ecf7cef80c24eca604b73d9b5e45b732 Mon Sep 17 00:00:00 2001 From: louie-github <30176969+louie-github@users.noreply.github.com> Date: Sat, 22 May 2021 02:09:48 +0800 Subject: [ThumbnailsConvertor] Support conversion to `png` and make it the default (#333) PNG, being a lossless format, should be a better default here compared to JPG since we won't be compressing to a lossy format and losing some of the original image data PNG is also supported for embedding in all the formats similar to JPEG Authored by: louie-github --- yt_dlp/postprocessor/ffmpeg.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index b15610829..f2e3559a5 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -849,24 +849,30 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): info['__files_to_move'].pop(thumbnail_filename), 'webp') def convert_thumbnail(self, thumbnail_filename, ext): - if ext != 'jpg': - raise FFmpegPostProcessorError('Only conversion to jpg is currently supported') + if ext == 'jpg': + format_name = 'JPEG' + opts = ['-bsf:v', 'mjpeg2jpeg'] + elif ext == 'png': + format_name = 'PNG' + opts = [] + else: + raise FFmpegPostProcessorError('Only conversion to either jpg or png is currently supported') # NB: % is supposed to be escaped with %% but this does not work # for input files so working around with standard substitution escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) - escaped_thumbnail_jpg_filename = replace_extension(escaped_thumbnail_filename, 'jpg') - self.to_screen('Converting thumbnail "%s" to JPEG' % escaped_thumbnail_filename) - self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_jpg_filename, ['-bsf:v', 'mjpeg2jpeg']) - thumbnail_jpg_filename = replace_extension(thumbnail_filename, 'jpg') + escaped_thumbnail_conv_filename = replace_extension(escaped_thumbnail_filename, ext) + self.to_screen('Converting thumbnail "%s" to %s' % (escaped_thumbnail_filename, format_name)) + self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_conv_filename, opts) + thumbnail_conv_filename = replace_extension(thumbnail_filename, ext) # Rename back to unescaped os.rename(encodeFilename(escaped_thumbnail_filename), encodeFilename(thumbnail_filename)) - os.rename(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename)) - return thumbnail_jpg_filename + os.rename(encodeFilename(escaped_thumbnail_conv_filename), encodeFilename(thumbnail_conv_filename)) + return thumbnail_conv_filename def run(self, info): - if self.format != 'jpg': - raise FFmpegPostProcessorError('Only conversion to jpg is currently supported') + if self.format not in ('jpg', 'png'): + raise FFmpegPostProcessorError('Only conversion to either jpg or png is currently supported') files_to_delete = [] has_thumbnail = False -- cgit v1.2.3 From 857f63136d4f46b18353f8e50b07ff1394b79695 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 22 May 2021 13:08:12 +0530 Subject: [videoconvertor] Generalize with remuxer and allow conditional recoding --- yt_dlp/postprocessor/ffmpeg.py | 71 +++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 39 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index f2e3559a5..810c9cb86 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -399,62 +399,55 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): return [path], information -class FFmpegVideoRemuxerPP(FFmpegPostProcessor): +class FFmpegVideoConvertorPP(FFmpegPostProcessor): + _action = 'converting' + def __init__(self, downloader=None, preferedformat=None): - super(FFmpegVideoRemuxerPP, self).__init__(downloader) + super(FFmpegVideoConvertorPP, self).__init__(downloader) self._preferedformats = preferedformat.lower().split('/') - def run(self, information): - path = information['filepath'] - sourceext, targetext = information['ext'].lower(), None + def _target_ext(self, source_ext): for pair in self._preferedformats: kv = pair.split('>') - if len(kv) == 1 or kv[0].strip() == sourceext: - targetext = kv[-1].strip() - break + if len(kv) == 1 or kv[0].strip() == source_ext: + return kv[-1].strip() + @staticmethod + def _options(target_ext): + if target_ext == 'avi': + return ['-c:v', 'libxvid', '-vtag', 'XVID'] + return [] + + def run(self, information): + path = information['filepath'] + target_ext = self._target_ext(information['ext'].lower()) _skip_msg = ( - 'could not find a mapping for %s' if not targetext - else 'already is in target format %s' if sourceext == targetext + 'could not find a mapping for %s' if not target_ext + else 'already is in target format %s' if source_ext == target_ext else None) if _skip_msg: - self.to_screen('Not remuxing media file %s; %s' % (path, _skip_msg % sourceext)) + self.to_screen('Not %s media file %s; %s' % (self._action, path, _skip_msg % source_ext)) return [], information - options = ['-c', 'copy', '-map', '0', '-dn'] - if targetext in ['mp4', 'm4a', 'mov']: - options.extend(['-movflags', '+faststart']) prefix, sep, oldext = path.rpartition('.') - outpath = prefix + sep + targetext - self.to_screen('Remuxing video from %s to %s; Destination: %s' % (sourceext, targetext, outpath)) - self.run_ffmpeg(path, outpath, options) + outpath = prefix + sep + target_ext + self.to_screen('%s video from %s to %s; Destination: %s' % (self._action.title(), source_ext, target_ext, outpath)) + self.run_ffmpeg(path, outpath, self._options(target_ext)) + information['filepath'] = outpath - information['format'] = targetext - information['ext'] = targetext + information['format'] = information['ext'] = target_ext return [path], information -class FFmpegVideoConvertorPP(FFmpegPostProcessor): - def __init__(self, downloader=None, preferedformat=None): - super(FFmpegVideoConvertorPP, self).__init__(downloader) - self._preferedformat = preferedformat +class FFmpegVideoRemuxerPP(FFmpegVideoConvertorPP): + _action = 'remuxing' - def run(self, information): - path = information['filepath'] - if information['ext'] == self._preferedformat: - self.to_screen('Not converting video file %s - already is in target format %s' % (path, self._preferedformat)) - return [], information - options = [] - if self._preferedformat == 'avi': - options.extend(['-c:v', 'libxvid', '-vtag', 'XVID']) - prefix, sep, ext = path.rpartition('.') - outpath = prefix + sep + self._preferedformat - self.to_screen('Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath) - self.run_ffmpeg(path, outpath, options) - information['filepath'] = outpath - information['format'] = self._preferedformat - information['ext'] = self._preferedformat - return [path], information + @staticmethod + def _options(target_ext): + options = ['-c', 'copy', '-map', '0', '-dn'] + if target_ext in ['mp4', 'm4a', 'mov']: + options.extend(['-movflags', '+faststart']) + return options class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): -- cgit v1.2.3 From 81a23040ebf330a87a7eb842aa76884d5fc6e504 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 22 May 2021 13:54:12 +0530 Subject: [cleanup] Refactor ffmpeg convertors --- yt_dlp/postprocessor/ffmpeg.py | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 810c9cb86..ea728be37 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -290,13 +290,12 @@ class FFmpegPostProcessor(PostProcessor): class FFmpegExtractAudioPP(FFmpegPostProcessor): - COMMON_AUDIO_EXTENSIONS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') + COMMON_AUDIO_EXTS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') + SUPPORTED_EXTS = ('best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav') def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): FFmpegPostProcessor.__init__(self, downloader) - if preferredcodec is None: - preferredcodec = 'best' - self._preferredcodec = preferredcodec + self._preferredcodec = preferredcodec or 'best' self._preferredquality = preferredquality self._nopostoverwrites = nopostoverwrites @@ -315,7 +314,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): path = information['filepath'] orig_ext = information['ext'] - if self._preferredcodec == 'best' and orig_ext in self.COMMON_AUDIO_EXTENSIONS: + if self._preferredcodec == 'best' and orig_ext in self.COMMON_AUDIO_EXTS: self.to_screen('Skipping audio extraction since the file is already in a common audio format') return [], information @@ -400,6 +399,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegVideoConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('mp4', 'mkv', 'flv', 'webm', 'mov', 'avi', 'mp3', 'mka', 'm4a', 'ogg', 'opus') + FORMAT_RE = re.compile(r'{0}(?:/{0})*$'.format(r'(?:\w+>)?(?:%s)' % '|'.join(SUPPORTED_EXTS))) _action = 'converting' def __init__(self, downloader=None, preferedformat=None): @@ -419,14 +420,14 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): return [] def run(self, information): - path = information['filepath'] - target_ext = self._target_ext(information['ext'].lower()) + path, source_ext = information['filepath'], information['ext'].lower() + target_ext = self._target_ext(source_ext) _skip_msg = ( 'could not find a mapping for %s' if not target_ext else 'already is in target format %s' if source_ext == target_ext else None) if _skip_msg: - self.to_screen('Not %s media file %s; %s' % (self._action, path, _skip_msg % source_ext)) + self.to_screen('Not %s media file "%s"; %s' % (self._action, path, _skip_msg % source_ext)) return [], information prefix, sep, oldext = path.rpartition('.') @@ -708,6 +709,8 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor): class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('srt', 'vtt', 'ass', 'lrc') + def __init__(self, downloader=None, format=None): super(FFmpegSubtitlesConvertorPP, self).__init__(downloader) self.format = format @@ -816,6 +819,8 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('jpg', 'png') + def __init__(self, downloader=None, format=None): super(FFmpegThumbnailsConvertorPP, self).__init__(downloader) self.format = format @@ -841,31 +846,29 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): info['__files_to_move'][webp_filename] = replace_extension( info['__files_to_move'].pop(thumbnail_filename), 'webp') - def convert_thumbnail(self, thumbnail_filename, ext): - if ext == 'jpg': - format_name = 'JPEG' - opts = ['-bsf:v', 'mjpeg2jpeg'] - elif ext == 'png': - format_name = 'PNG' - opts = [] - else: - raise FFmpegPostProcessorError('Only conversion to either jpg or png is currently supported') + @staticmethod + def _options(target_ext): + if target_ext == 'jpg': + return ['-bsf:v', 'mjpeg2jpeg'] + return [] + + def convert_thumbnail(self, thumbnail_filename, target_ext): # NB: % is supposed to be escaped with %% but this does not work # for input files so working around with standard substitution escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) - escaped_thumbnail_conv_filename = replace_extension(escaped_thumbnail_filename, ext) - self.to_screen('Converting thumbnail "%s" to %s' % (escaped_thumbnail_filename, format_name)) - self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_conv_filename, opts) - thumbnail_conv_filename = replace_extension(thumbnail_filename, ext) + escaped_thumbnail_conv_filename = replace_extension(escaped_thumbnail_filename, target_ext) + + self.to_screen('Converting thumbnail "%s" to %s' % (escaped_thumbnail_filename, target_ext)) + self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_conv_filename, self._options(target_ext)) + # Rename back to unescaped + thumbnail_conv_filename = replace_extension(thumbnail_filename, target_ext) os.rename(encodeFilename(escaped_thumbnail_filename), encodeFilename(thumbnail_filename)) os.rename(encodeFilename(escaped_thumbnail_conv_filename), encodeFilename(thumbnail_conv_filename)) return thumbnail_conv_filename def run(self, info): - if self.format not in ('jpg', 'png'): - raise FFmpegPostProcessorError('Only conversion to either jpg or png is currently supported') files_to_delete = [] has_thumbnail = False -- cgit v1.2.3 From 337e0c62f894722e9c268b14d02a85b84c96024d Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 29 May 2021 01:39:07 +0530 Subject: [embedthumbnail] Correctly escape filename Closes #352 The approach in [1] is faulty as can be seen in the test cases 1. https://github.com/ytdl-org/youtube-dl/commit/bff857a8af696e701482208617bf0b7564951326 --- yt_dlp/postprocessor/ffmpeg.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index ea728be37..d9f816b04 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -853,19 +853,12 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): return [] def convert_thumbnail(self, thumbnail_filename, target_ext): - # NB: % is supposed to be escaped with %% but this does not work - # for input files so working around with standard substitution - escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') - os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) - escaped_thumbnail_conv_filename = replace_extension(escaped_thumbnail_filename, target_ext) - - self.to_screen('Converting thumbnail "%s" to %s' % (escaped_thumbnail_filename, target_ext)) - self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_conv_filename, self._options(target_ext)) - - # Rename back to unescaped thumbnail_conv_filename = replace_extension(thumbnail_filename, target_ext) - os.rename(encodeFilename(escaped_thumbnail_filename), encodeFilename(thumbnail_filename)) - os.rename(encodeFilename(escaped_thumbnail_conv_filename), encodeFilename(thumbnail_conv_filename)) + + self.to_screen('Converting thumbnail "%s" to %s' % (thumbnail_filename, target_ext)) + self.real_run_ffmpeg( + [(thumbnail_filename, ['-f', 'image2', '-pattern_type', 'none'])], + [(thumbnail_conv_filename.replace('%', '%%'), self._options(target_ext))]) return thumbnail_conv_filename def run(self, info): -- cgit v1.2.3 From 324ad82006748ebfe4b3fa8f67f160eb000ee6eb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 8 Jun 2021 14:23:56 +0530 Subject: [utils] Generalize `traverse_dict` to `traverse_obj` --- yt_dlp/postprocessor/ffmpeg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index d9f816b04..374da8c02 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -23,7 +23,7 @@ from ..utils import ( ISO639Utils, process_communicate_or_kill, replace_extension, - traverse_dict, + traverse_obj, ) @@ -229,7 +229,7 @@ class FFmpegPostProcessor(PostProcessor): def get_stream_number(self, path, keys, value): streams = self.get_metadata_object(path)['streams'] num = next( - (i for i, stream in enumerate(streams) if traverse_dict(stream, keys, casesense=False) == value), + (i for i, stream in enumerate(streams) if traverse_obj(stream, keys, casesense=False) == value), None) return num, len(streams) -- cgit v1.2.3 From 8326b00aabc332cad3edec246fe5353bea069cb0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 13 Jun 2021 01:32:19 +0530 Subject: Allow `images` formats Necessary for #343. * They are identified by `vcodec=acodec='none'` * These formats show as the worst in `-F` * Any postprocessor that expects audio/video will be skipped * `b*` and all related selectors will skip such formats * This commit also does not add any selector for downloading such formats. They have to be explicitly requested by the `format_id`. Implementation of a selector is left for when #389 is resolved --- yt_dlp/postprocessor/ffmpeg.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 374da8c02..273f1b763 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -310,6 +310,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): except FFmpegPostProcessorError as err: raise AudioConversionError(err.msg) + @PostProcessor._restrict_to(images=False) def run(self, information): path = information['filepath'] orig_ext = information['ext'] @@ -419,6 +420,7 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): return ['-c:v', 'libxvid', '-vtag', 'XVID'] return [] + @PostProcessor._restrict_to(images=False) def run(self, information): path, source_ext = information['filepath'], information['ext'].lower() target_ext = self._target_ext(source_ext) @@ -456,6 +458,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): super(FFmpegEmbedSubtitlePP, self).__init__(downloader) self._already_have_subtitle = already_have_subtitle + @PostProcessor._restrict_to(images=False) def run(self, information): if information['ext'] not in ('mp4', 'webm', 'mkv'): self.to_screen('Subtitles can only be embedded in mp4, webm or mkv files') @@ -523,6 +526,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False) def run(self, info): metadata = {} @@ -625,6 +629,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): class FFmpegMergerPP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False) def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') @@ -657,6 +662,7 @@ class FFmpegMergerPP(FFmpegPostProcessor): class FFmpegFixupStretchedPP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False, audio=False) def run(self, info): stretched_ratio = info.get('stretched_ratio') if stretched_ratio is None or stretched_ratio == 1: @@ -676,6 +682,7 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor): class FFmpegFixupM4aPP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False, video=False) def run(self, info): if info.get('container') != 'm4a_dash': return [], info @@ -694,6 +701,7 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor): class FFmpegFixupM3u8PP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False) def run(self, info): filename = info['filepath'] if self.get_audio_codec(filename) == 'aac': @@ -805,6 +813,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): ['-ss', compat_str(chapter['start_time']), '-t', compat_str(chapter['end_time'] - chapter['start_time'])]) + @PostProcessor._restrict_to(images=False) def run(self, info): chapters = info.get('chapters') or [] if not chapters: -- cgit v1.2.3 From fd7cfb6444272678b9bada28ed3dfa68535dc85e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 20 Jun 2021 03:49:23 +0530 Subject: [cleanup] Refactor fixup --- yt_dlp/postprocessor/ffmpeg.py | 54 +++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 273f1b763..49685951e 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -661,58 +661,42 @@ class FFmpegMergerPP(FFmpegPostProcessor): return True -class FFmpegFixupStretchedPP(FFmpegPostProcessor): - @PostProcessor._restrict_to(images=False, audio=False) - def run(self, info): - stretched_ratio = info.get('stretched_ratio') - if stretched_ratio is None or stretched_ratio == 1: - return [], info - - filename = info['filepath'] +class FFmpegFixupPostProcessor(FFmpegPostProcessor): + def _fixup(self, msg, filename, options): temp_filename = prepend_extension(filename, 'temp') - options = ['-c', 'copy', '-map', '0', '-dn', '-aspect', '%f' % stretched_ratio] - self.to_screen('Fixing aspect ratio in "%s"' % filename) + self.to_screen('{msg} of "{filename}"') self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + +class FFmpegFixupStretchedPP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False, audio=False) + def run(self, info): + stretched_ratio = info.get('stretched_ratio') + if stretched_ratio not in (None, 1): + self._fixup('Fixing aspect ratio', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-aspect', '%f' % stretched_ratio]) return [], info -class FFmpegFixupM4aPP(FFmpegPostProcessor): +class FFmpegFixupM4aPP(FFmpegFixupPostProcessor): @PostProcessor._restrict_to(images=False, video=False) def run(self, info): - if info.get('container') != 'm4a_dash': - return [], info - - filename = info['filepath'] - temp_filename = prepend_extension(filename, 'temp') - - options = ['-c', 'copy', '-map', '0', '-dn', '-f', 'mp4'] - self.to_screen('Correcting container in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) - - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) - + if info.get('container') == 'm4a_dash': + self._fixup('Correcting container', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4']) return [], info -class FFmpegFixupM3u8PP(FFmpegPostProcessor): +class FFmpegFixupM3u8PP(FFmpegFixupPostProcessor): @PostProcessor._restrict_to(images=False) def run(self, info): - filename = info['filepath'] - if self.get_audio_codec(filename) == 'aac': - temp_filename = prepend_extension(filename, 'temp') - - options = ['-c', 'copy', '-map', '0', '-dn', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'] - self.to_screen('Fixing malformed AAC bitstream in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) - - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + if self.get_audio_codec(info['filepath']) == 'aac': + self._fixup('Fixing malformed AAC bitstream', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']) return [], info -- cgit v1.2.3 From f89b3e2d7ab7a6e28fc1d9975aa4e998b165c090 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 20 Jun 2021 04:15:19 +0530 Subject: Skip fixup of existing files and add `--fixup force` to force it --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 49685951e..4685288a7 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -665,7 +665,7 @@ class FFmpegFixupPostProcessor(FFmpegPostProcessor): def _fixup(self, msg, filename, options): temp_filename = prepend_extension(filename, 'temp') - self.to_screen('{msg} of "{filename}"') + self.to_screen(f'{msg} of "{filename}"') self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) -- cgit v1.2.3 From e36d50c5dd35973c090f87df05d4e94963e8036c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 21 Jun 2021 22:53:17 +0530 Subject: [websockets] Add `WebSocketFragmentFD` (#399) Necessary for #392 Co-authored by: nao20010128nao, pukkandan --- yt_dlp/postprocessor/ffmpeg.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 4685288a7..83714358e 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -700,6 +700,35 @@ class FFmpegFixupM3u8PP(FFmpegFixupPostProcessor): return [], info +class FFmpegFixupTimestampPP(FFmpegFixupPostProcessor): + + def __init__(self, downloader=None, trim=0.001): + # "trim" should be used when the video contains unintended packets + super(FFmpegFixupTimestampPP, self).__init__(downloader) + assert isinstance(trim, (int, float)) + self.trim = str(trim) + + @PostProcessor._restrict_to(images=False) + def run(self, info): + required_version = '4.4' + if is_outdated_version(self._versions[self.basename], required_version): + self.report_warning( + 'A re-encode is needed to fix timestamps in older versions of ffmpeg. ' + f'Please install ffmpeg {required_version} or later to fixup without re-encoding') + opts = ['-vf', 'setpts=PTS-STARTPTS'] + else: + opts = ['-c', 'copy', '-bsf', 'setts=ts=TS-STARTPTS'] + self._fixup('Fixing frame timestamp', info['filepath'], opts + ['-map', '0', '-dn', '-ss', self.trim]) + return [], info + + +class FFmpegFixupDurationPP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False) + def run(self, info): + self._fixup('Fixing video duration', info['filepath'], ['-c', 'copy', '-map', '0', '-dn']) + return [], info + + class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): SUPPORTED_EXTS = ('srt', 'vtt', 'ass', 'lrc') -- cgit v1.2.3 From 15a4fd53d3a31b1aabbd6f0ad29f981cd27a06a5 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 25 Jun 2021 05:36:35 +0530 Subject: [thumbnailsconvertor] Treat `jpeg` as `jpg` --- yt_dlp/postprocessor/ffmpeg.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 83714358e..0d5e78f3d 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -896,6 +896,8 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): _, thumbnail_ext = os.path.splitext(original_thumbnail) if thumbnail_ext: thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext == 'jpeg': + thumbnail_ext = 'jpg' if thumbnail_ext == self.format: self.to_screen('Thumbnail "%s" is already in the requested format' % original_thumbnail) continue -- cgit v1.2.3 From 6606817a86b96cc66aaa1d567b7bfce0c75500a2 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 11 Jul 2021 03:29:44 +0530 Subject: [utils] Add `variadic` --- yt_dlp/postprocessor/ffmpeg.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 0d5e78f3d..fcc32ca03 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -24,6 +24,7 @@ from ..utils import ( process_communicate_or_kill, replace_extension, traverse_obj, + variadic, ) @@ -533,15 +534,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor): def add(meta_list, info_list=None): if not meta_list: return - if not info_list: - info_list = meta_list - if not isinstance(meta_list, (list, tuple)): - meta_list = (meta_list,) - if not isinstance(info_list, (list, tuple)): - info_list = (info_list,) - for info_f in info_list: + for info_f in variadic(info_list or meta_list): if isinstance(info.get(info_f), (compat_str, compat_numeric_types)): - for meta_f in meta_list: + for meta_f in variadic(meta_list): metadata[meta_f] = info[info_f] break -- cgit v1.2.3 From 7dde84f3c9e74e81a7b4de7d96e512e914344118 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 11 Jul 2021 00:26:35 +0530 Subject: [FFmpegMetadata] Add language of each stream and some refactoring --- yt_dlp/postprocessor/ffmpeg.py | 68 +++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 28 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index fcc32ca03..85cd0288a 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import io +import itertools import os import subprocess import time @@ -243,7 +244,7 @@ class FFmpegPostProcessor(PostProcessor): self.check_version() oldest_mtime = min( - os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts) + os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts if path) cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] # avconv does not have repeat option @@ -262,8 +263,9 @@ class FFmpegPostProcessor(PostProcessor): + [encodeFilename(self._ffmpeg_filename_argument(file), True)]) for arg_type, path_opts in (('i', input_path_opts), ('o', output_path_opts)): - cmd += [arg for i, o in enumerate(path_opts) - for arg in make_args(o[0], o[1], arg_type, i + 1)] + cmd += itertools.chain.from_iterable( + make_args(path, list(opts), arg_type, i + 1) + for i, (path, opts) in enumerate(path_opts) if path) self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) @@ -274,7 +276,8 @@ class FFmpegPostProcessor(PostProcessor): self.report_error(stderr) raise FFmpegPostProcessorError(stderr.split('\n')[-1]) for out_path, _ in output_path_opts: - self.try_utime(out_path, oldest_mtime, oldest_mtime) + if out_path: + self.try_utime(out_path, oldest_mtime, oldest_mtime) return stderr.decode('utf-8', 'replace') def run_ffmpeg(self, path, out_path, opts): @@ -527,6 +530,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): + + @staticmethod + def _options(target_ext): + yield from ('-map', '0', '-dn') + if target_ext == 'm4a': + yield from ('-vn', '-acodec', 'copy') + else: + yield from ('-c', 'copy') + @PostProcessor._restrict_to(images=False) def run(self, info): metadata = {} @@ -565,22 +577,17 @@ class FFmpegMetadataPP(FFmpegPostProcessor): for key in filter(lambda k: k.startswith(prefix), info.keys()): add(key[len(prefix):], key) - if not metadata: - self.to_screen('There isn\'t any metadata to add') - return [], info - - filename = info['filepath'] - temp_filename = prepend_extension(filename, 'temp') - in_filenames = [filename] - options = ['-map', '0', '-dn'] - - if info['ext'] == 'm4a': - options.extend(['-vn', '-acodec', 'copy']) - else: - options.extend(['-c', 'copy']) + filename, metadata_filename = info['filepath'], None + options = [('-metadata', f'{name}={value}') for name, value in metadata.items()] - for name, value in metadata.items(): - options.extend(['-metadata', '%s=%s' % (name, value)]) + stream_idx = 0 + for fmt in info.get('requested_formats') or []: + stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1 + if fmt.get('language'): + lang = ISO639Utils.short2long(fmt['language']) or fmt['language'] + options.extend(('-metadata:s:%d' % (stream_idx + i), 'language=%s' % lang) + for i in range(stream_count)) + stream_idx += stream_count chapters = info.get('chapters', []) if chapters: @@ -598,24 +605,29 @@ class FFmpegMetadataPP(FFmpegPostProcessor): if chapter_title: metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) f.write(metadata_file_content) - in_filenames.append(metadata_filename) - options.extend(['-map_metadata', '1']) + options.append(('-map_metadata', '1')) if ('no-attach-info-json' not in self.get_param('compat_opts', []) and '__infojson_filename' in info and info['ext'] in ('mkv', 'mka')): - old_stream, new_stream = self.get_stream_number( - filename, ('tags', 'mimetype'), 'application/json') + old_stream, new_stream = self.get_stream_number(filename, ('tags', 'mimetype'), 'application/json') if old_stream is not None: - options.extend(['-map', '-0:%d' % old_stream]) + options.append(('-map', '-0:%d' % old_stream)) new_stream -= 1 - options.extend([ + options.append(( '-attach', info['__infojson_filename'], '-metadata:s:%d' % new_stream, 'mimetype=application/json' - ]) + )) - self.to_screen('Adding metadata to \'%s\'' % filename) - self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options) + if not options: + self.to_screen('There isn\'t any metadata to add') + return [], info + + temp_filename = prepend_extension(filename, 'temp') + self.to_screen('Adding metadata to "%s"' % filename) + self.run_ffmpeg_multiple_files( + (filename, metadata_filename), temp_filename, + itertools.chain(self._options(info['ext']), *options)) if chapters: os.remove(metadata_filename) os.remove(encodeFilename(filename)) -- cgit v1.2.3 From 00034c146a2d8c84d7fc388c64eb29916105b754 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 11 Jul 2021 04:07:25 +0530 Subject: [embedthumbnail] Fix `_get_thumbnail_resolution` --- yt_dlp/postprocessor/ffmpeg.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 85cd0288a..eb5ae1737 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -235,12 +235,12 @@ class FFmpegPostProcessor(PostProcessor): None) return num, len(streams) - def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, **kwargs): return self.real_run_ffmpeg( [(path, []) for path in input_paths], - [(out_path, opts)]) + [(out_path, opts)], **kwargs) - def real_run_ffmpeg(self, input_path_opts, output_path_opts): + def real_run_ffmpeg(self, input_path_opts, output_path_opts, *, expected_retcodes=(0,)): self.check_version() oldest_mtime = min( @@ -270,7 +270,7 @@ class FFmpegPostProcessor(PostProcessor): self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) stdout, stderr = process_communicate_or_kill(p) - if p.returncode != 0: + if p.returncode not in variadic(expected_retcodes): stderr = stderr.decode('utf-8', 'replace').strip() if self.get_param('verbose', False): self.report_error(stderr) @@ -280,8 +280,8 @@ class FFmpegPostProcessor(PostProcessor): self.try_utime(out_path, oldest_mtime, oldest_mtime) return stderr.decode('utf-8', 'replace') - def run_ffmpeg(self, path, out_path, opts): - return self.run_ffmpeg_multiple_files([path], out_path, opts) + def run_ffmpeg(self, path, out_path, opts, **kwargs): + return self.run_ffmpeg_multiple_files([path], out_path, opts, **kwargs) def _ffmpeg_filename_argument(self, fn): # Always use 'file:' because the filename may contain ':' (ffmpeg -- cgit v1.2.3 From 89efdc15dd4dbdb4b51e82647637f33112156f61 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 4 Aug 2021 22:25:16 +0530 Subject: [ffpmeg] Allow `--ffmpeg-location` to be a file with different name --- yt_dlp/postprocessor/ffmpeg.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index eb5ae1737..bf6677239 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -111,19 +111,14 @@ class FFmpegPostProcessor(PostProcessor): return elif not os.path.isdir(location): basename = os.path.splitext(os.path.basename(location))[0] - if basename not in programs: - self.report_warning( - 'Cannot identify executable %s, its basename should be one of %s. ' - 'Continuing without ffmpeg.' % - (location, ', '.join(programs))) - self._versions = {} - return None - location = os.path.dirname(os.path.abspath(location)) + basename = next((p for p in programs if basename.startswith(p)), 'ffmpeg') + dirname = os.path.dirname(os.path.abspath(location)) if basename in ('ffmpeg', 'ffprobe'): prefer_ffmpeg = True self._paths = dict( - (p, os.path.join(location, p)) for p in programs) + (p, os.path.join(dirname, p)) for p in programs) + self._paths[basename] = location self._versions = dict( (p, get_ffmpeg_version(self._paths[p])) for p in programs) if self._versions is None: -- cgit v1.2.3 From 8c0ae192a42965f318644373d05060e720af79c7 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 9 Aug 2021 02:18:58 +0530 Subject: [ffmpeg] Fix `--ffmpeg-location` when directory is given Bug introduced in 89efdc15dd4dbdb4b51e82647637f33112156f61 Closes #654 --- yt_dlp/postprocessor/ffmpeg.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index bf6677239..24466f21c 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -109,7 +109,9 @@ class FFmpegPostProcessor(PostProcessor): 'Continuing without ffmpeg.' % (location)) self._versions = {} return - elif not os.path.isdir(location): + elif os.path.isdir(location): + dirname, basename = location, None + else: basename = os.path.splitext(os.path.basename(location))[0] basename = next((p for p in programs if basename.startswith(p)), 'ffmpeg') dirname = os.path.dirname(os.path.abspath(location)) @@ -118,7 +120,8 @@ class FFmpegPostProcessor(PostProcessor): self._paths = dict( (p, os.path.join(dirname, p)) for p in programs) - self._paths[basename] = location + if basename: + self._paths[basename] = location self._versions = dict( (p, get_ffmpeg_version(self._paths[p])) for p in programs) if self._versions is None: -- cgit v1.2.3 From a8731fcc1d11afe0c6c3e349d0c85d32ed17488a Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 11 Aug 2021 13:42:23 +0530 Subject: minor bugfixes bugs due to be2fc5b212338d89d9c139cb463f785e797d1ad3, e9f4ccd19eb92621970b518fb5984b8aef52bdc8 --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 24466f21c..1d21ffaf2 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -592,7 +592,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): metadata_filename = replace_extension(filename, 'meta') with io.open(metadata_filename, 'wt', encoding='utf-8') as f: def ffmpeg_escape(text): - return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text) + return re.sub(r'([\\=;#\n])', r'\\\1', text) metadata_file_content = ';FFMETADATA1\n' for chapter in chapters: -- cgit v1.2.3 From 37242e56f22af4dfc45cf34784f25782f52610fc Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 13 Aug 2021 20:40:13 +0530 Subject: Fix bug during subtitle conversion --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 1d21ffaf2..be6cc9f09 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -802,7 +802,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): } info['__files_to_move'][new_file] = replace_extension( - info['__files_to_move'][old_file], new_ext) + info['__files_to_move'][sub['filepath']], new_ext) return sub_filenames, info -- cgit v1.2.3 From d75201a873a413d73f12748e5710f000e9f727da Mon Sep 17 00:00:00 2001 From: Paul Wrubel Date: Thu, 26 Aug 2021 21:27:20 -0500 Subject: Use `os.replace` where applicable (#793) When using ```py os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) ``` the `os.remove` need not be atomic and so can be executed arbitrarily compared to the immediately following rename call. It is better to use `os.replace` instead Authored by: paulwrubel --- yt_dlp/postprocessor/ffmpeg.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index be6cc9f09..b66a0b445 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -520,8 +520,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): temp_filename = prepend_extension(filename, 'temp') self.to_screen('Embedding subtitles in "%s"' % filename) self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + os.replace(temp_filename, filename) files_to_delete = [] if self._already_have_subtitle else sub_filenames return files_to_delete, information @@ -628,8 +627,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): itertools.chain(self._options(info['ext']), *options)) if chapters: os.remove(metadata_filename) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + os.replace(temp_filename, filename) return [], info @@ -673,8 +671,7 @@ class FFmpegFixupPostProcessor(FFmpegPostProcessor): self.to_screen(f'{msg} of "{filename}"') self.run_ffmpeg(filename, temp_filename, options) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + os.replace(temp_filename, filename) class FFmpegFixupStretchedPP(FFmpegFixupPostProcessor): @@ -866,9 +863,7 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): if thumbnail_ext != 'webp' and self.is_webp(thumbnail_filename): self.to_screen('Correcting thumbnail "%s" extension to webp' % thumbnail_filename) webp_filename = replace_extension(thumbnail_filename, 'webp') - if os.path.exists(webp_filename): - os.remove(webp_filename) - os.rename(encodeFilename(thumbnail_filename), encodeFilename(webp_filename)) + os.replace(thumbnail_filename, webp_filename) info['thumbnails'][idx]['filepath'] = webp_filename info['__files_to_move'][webp_filename] = replace_extension( info['__files_to_move'].pop(thumbnail_filename), 'webp') -- cgit v1.2.3 From 8e25d624df003d691be922488d6ab7007f75333d Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 1 Sep 2021 08:45:56 +0530 Subject: [EmbedSubtitle] Continue even if some files are missing --- yt_dlp/postprocessor/ffmpeg.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index b66a0b445..7537d5db4 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -478,6 +478,9 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): mp4_ass_warn = False for lang, sub_info in subtitles.items(): + if not os.path.exists(information.get('filepath', '')): + self.report_warning(f'Skipping embedding {lang} subtitle because the file is missing') + continue sub_ext = sub_info['ext'] if sub_ext == 'json': self.report_warning('JSON subtitles cannot be embedded') -- cgit v1.2.3 From 7a340e0df352bf97da7a7fd238f7d705afbd9c6a Mon Sep 17 00:00:00 2001 From: Nil Admirari <50202386+nihil-admirari@users.noreply.github.com> Date: Wed, 1 Sep 2021 20:55:16 +0000 Subject: Native SponsorBlock implementation and related improvements (#360) SponsorBlock options: * The fetched sponsor sections are written to infojson * `--sponsorblock-remove` removes specified chapters from file * `--sponsorblock-mark` marks the specified sponsor sections as chapters * `--sponsorblock-chapter-title` to specify sponsor chapter template * `--sponsorblock-api` to use a different API Related improvements: * Split `--embed-chapters` from `--embed-metadata` * Add `--remove-chapters` to remove arbitrary chapters * Add `--force-keyframes-at-cuts` for more accurate cuts when removing and splitting chapters Deprecates all `--sponskrub` options Authored by: nihil-admirari, pukkandan --- yt_dlp/postprocessor/ffmpeg.py | 174 +++++++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 49 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 7537d5db4..806334645 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -8,22 +8,22 @@ import time import re import json - from .common import AudioConversionError, PostProcessor from ..compat import compat_str, compat_numeric_types from ..utils import ( + dfxp2srt, encodeArgument, encodeFilename, get_exe_version, is_outdated_version, + ISO639Utils, + orderedSet, PostProcessingError, prepend_extension, - shell_quote, - dfxp2srt, - ISO639Utils, process_communicate_or_kill, replace_extension, + shell_quote, traverse_obj, variadic, ) @@ -281,7 +281,8 @@ class FFmpegPostProcessor(PostProcessor): def run_ffmpeg(self, path, out_path, opts, **kwargs): return self.run_ffmpeg_multiple_files([path], out_path, opts, **kwargs) - def _ffmpeg_filename_argument(self, fn): + @staticmethod + def _ffmpeg_filename_argument(fn): # Always use 'file:' because the filename may contain ':' (ffmpeg # interprets that as a protocol) or can start with '-' (-- is broken in # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details) @@ -290,6 +291,62 @@ class FFmpegPostProcessor(PostProcessor): return fn return 'file:' + fn if fn != '-' else fn + @staticmethod + def _quote_for_ffmpeg(string): + # See https://ffmpeg.org/ffmpeg-utils.html#toc-Quoting-and-escaping + # A sequence of '' produces '\'''\''; + # final replace removes the empty '' between \' \'. + string = string.replace("'", r"'\''").replace("'''", "'") + # Handle potential ' at string boundaries. + string = string[1:] if string[0] == "'" else "'" + string + return string[:-1] if string[-1] == "'" else string + "'" + + def force_keyframes(self, filename, timestamps): + timestamps = orderedSet(timestamps) + if timestamps[0] == 0: + timestamps = timestamps[1:] + keyframe_file = prepend_extension(filename, 'keyframes.temp') + self.to_screen(f'Re-encoding "{filename}" with appropriate keyframes') + self.run_ffmpeg(filename, keyframe_file, ['-force_key_frames', ','.join( + f'{t:.6f}' for t in timestamps)]) + return keyframe_file + + def concat_files(self, in_files, out_file, concat_opts=None): + """ + Use concat demuxer to concatenate multiple files having identical streams. + + Only inpoint, outpoint, and duration concat options are supported. + See https://ffmpeg.org/ffmpeg-formats.html#concat-1 for details + """ + concat_file = f'{out_file}.concat' + self.write_debug(f'Writing concat spec to {concat_file}') + with open(concat_file, 'wt', encoding='utf-8') as f: + f.writelines(self._concat_spec(in_files, concat_opts)) + + out_flags = ['-c', 'copy'] + if out_file.rpartition('.')[-1] in ('mp4', 'mov'): + # For some reason, '-c copy' is not enough to copy subtitles + out_flags.extend(['-c:s', 'mov_text', '-movflags', '+faststart']) + + try: + self.real_run_ffmpeg( + [(concat_file, ['-hide_banner', '-nostdin', '-f', 'concat', '-safe', '0'])], + [(out_file, out_flags)]) + finally: + os.remove(concat_file) + + @classmethod + def _concat_spec(cls, in_files, concat_opts=None): + if concat_opts is None: + concat_opts = [{}] * len(in_files) + yield 'ffconcat version 1.0\n' + for file, opts in zip(in_files, concat_opts): + yield f'file {cls._quote_for_ffmpeg(cls._ffmpeg_filename_argument(file))}\n' + # Iterate explicitly to yield the following directives in order, ignoring the rest. + for directive in 'inpoint', 'outpoint', 'duration': + if directive in opts: + yield f'{directive} {opts[directive]}\n' + class FFmpegExtractAudioPP(FFmpegPostProcessor): COMMON_AUDIO_EXTS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') @@ -531,6 +588,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor): + def __init__(self, downloader, add_metadata=True, add_chapters=True): + FFmpegPostProcessor.__init__(self, downloader) + self._add_metadata = add_metadata + self._add_chapters = add_chapters + @staticmethod def _options(target_ext): yield from ('-map', '0', '-dn') @@ -541,6 +603,46 @@ class FFmpegMetadataPP(FFmpegPostProcessor): @PostProcessor._restrict_to(images=False) def run(self, info): + filename, metadata_filename = info['filepath'], None + options = [] + if self._add_chapters and info.get('chapters'): + metadata_filename = replace_extension(filename, 'meta') + options.extend(self._get_chapter_opts(info['chapters'], metadata_filename)) + if self._add_metadata: + options.extend(self._get_metadata_opts(info)) + + if not options: + self.to_screen('There isn\'t any metadata to add') + return [], info + + temp_filename = prepend_extension(filename, 'temp') + self.to_screen('Adding metadata to "%s"' % filename) + self.run_ffmpeg_multiple_files( + (filename, metadata_filename), temp_filename, + itertools.chain(self._options(info['ext']), *options)) + if metadata_filename: + os.remove(metadata_filename) + os.replace(temp_filename, filename) + return [], info + + @staticmethod + def _get_chapter_opts(chapters, metadata_filename): + with io.open(metadata_filename, 'wt', encoding='utf-8') as f: + def ffmpeg_escape(text): + return re.sub(r'([\\=;#\n])', r'\\\1', text) + + metadata_file_content = ';FFMETADATA1\n' + for chapter in chapters: + metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' + metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000) + metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000) + chapter_title = chapter.get('title') + if chapter_title: + metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) + f.write(metadata_file_content) + yield ('-map_metadata', '1') + + def _get_metadata_opts(self, info): metadata = {} def add(meta_list, info_list=None): @@ -577,61 +679,27 @@ class FFmpegMetadataPP(FFmpegPostProcessor): for key in filter(lambda k: k.startswith(prefix), info.keys()): add(key[len(prefix):], key) - filename, metadata_filename = info['filepath'], None - options = [('-metadata', f'{name}={value}') for name, value in metadata.items()] + for name, value in metadata.items(): + yield ('-metadata', f'{name}={value}') stream_idx = 0 for fmt in info.get('requested_formats') or []: stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1 if fmt.get('language'): lang = ISO639Utils.short2long(fmt['language']) or fmt['language'] - options.extend(('-metadata:s:%d' % (stream_idx + i), 'language=%s' % lang) - for i in range(stream_count)) + for i in range(stream_count): + yield ('-metadata:s:%d' % (stream_idx + i), 'language=%s' % lang) stream_idx += stream_count - chapters = info.get('chapters', []) - if chapters: - metadata_filename = replace_extension(filename, 'meta') - with io.open(metadata_filename, 'wt', encoding='utf-8') as f: - def ffmpeg_escape(text): - return re.sub(r'([\\=;#\n])', r'\\\1', text) - - metadata_file_content = ';FFMETADATA1\n' - for chapter in chapters: - metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' - metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000) - metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000) - chapter_title = chapter.get('title') - if chapter_title: - metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) - f.write(metadata_file_content) - options.append(('-map_metadata', '1')) - if ('no-attach-info-json' not in self.get_param('compat_opts', []) and '__infojson_filename' in info and info['ext'] in ('mkv', 'mka')): - old_stream, new_stream = self.get_stream_number(filename, ('tags', 'mimetype'), 'application/json') + old_stream, new_stream = self.get_stream_number(info['filepath'], ('tags', 'mimetype'), 'application/json') if old_stream is not None: - options.append(('-map', '-0:%d' % old_stream)) + yield ('-map', '-0:%d' % old_stream) new_stream -= 1 - options.append(( - '-attach', info['__infojson_filename'], - '-metadata:s:%d' % new_stream, 'mimetype=application/json' - )) - - if not options: - self.to_screen('There isn\'t any metadata to add') - return [], info - - temp_filename = prepend_extension(filename, 'temp') - self.to_screen('Adding metadata to "%s"' % filename) - self.run_ffmpeg_multiple_files( - (filename, metadata_filename), temp_filename, - itertools.chain(self._options(info['ext']), *options)) - if chapters: - os.remove(metadata_filename) - os.replace(temp_filename, filename) - return [], info + yield ('-attach', info['__infojson_filename'], + '-metadata:s:%d' % new_stream, 'mimetype=application/json') class FFmpegMergerPP(FFmpegPostProcessor): @@ -808,6 +876,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): class FFmpegSplitChaptersPP(FFmpegPostProcessor): + def __init__(self, downloader, force_keyframes=False): + FFmpegPostProcessor.__init__(self, downloader) + self._force_keyframes = force_keyframes def _prepare_filename(self, number, chapter, info): info = info.copy() @@ -835,13 +906,18 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor): def run(self, info): chapters = info.get('chapters') or [] if not chapters: - self.report_warning('Chapter information is unavailable') + self.to_screen('Chapter information is unavailable') return [], info + in_file = info['filepath'] + if self._force_keyframes and len(chapters) > 1: + in_file = self.force_keyframes(in_file, (c['start_time'] for c in chapters)) self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters)) for idx, chapter in enumerate(chapters): destination, opts = self._ffmpeg_args_for_chapter(idx + 1, chapter, info) - self.real_run_ffmpeg([(info['filepath'], opts)], [(destination, ['-c', 'copy'])]) + self.real_run_ffmpeg([(in_file, opts)], [(destination, ['-c', 'copy'])]) + if in_file != info['filepath']: + os.remove(in_file) return [], info -- cgit v1.2.3 From 165efb823b3a8a6a6788cfe23e6b93dfbe150568 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 4 Sep 2021 01:37:41 +0530 Subject: [ModifyChapters] fixes (See desc) * [docs] Fix typo * Do not enable `sponskrub` by default * Fix `--force-keyframes-at-cuts` * Don't embed subtitles if the video has been cut. Previously, running `--remove-chapters` with `--embed-subs` multiple times caused repeated cuts and out-of-sync subtitles * Store `_real_duration` to prevent running ffprobe multiple times --- yt_dlp/postprocessor/ffmpeg.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 806334645..25488e58b 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -15,6 +15,7 @@ from ..utils import ( dfxp2srt, encodeArgument, encodeFilename, + float_or_none, get_exe_version, is_outdated_version, ISO639Utils, @@ -233,6 +234,23 @@ class FFmpegPostProcessor(PostProcessor): None) return num, len(streams) + def _get_real_video_duration(self, info, fatal=True): + try: + if '_real_duration' not in info: + info['_real_duration'] = float_or_none( + traverse_obj(self.get_metadata_object(info['filepath']), ('format', 'duration'))) + if not info['_real_duration']: + raise PostProcessingError('ffprobe returned empty duration') + except PostProcessingError as e: + if fatal: + raise PostProcessingError(f'Unable to determine video duration; {e}') + return info.setdefault('_real_duration', None) + + def _duration_mismatch(self, d1, d2): + if not d1 or not d2: + return None + return abs(d1 - d2) > 1 + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, **kwargs): return self.real_run_ffmpeg( [(path, []) for path in input_paths], @@ -528,6 +546,10 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): return [], information filename = information['filepath'] + if self._duration_mismatch( + self._get_real_video_duration(information, False), information['duration']): + self.to_screen(f'Skipping {self.pp_key()} since the real and expected durations mismatch') + return [], information ext = information['ext'] sub_langs, sub_names, sub_filenames = [], [], [] -- cgit v1.2.3 From a21e0ab1a1a03f82517cd8cec4b9a2b4d6b81ac3 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 22 Sep 2021 19:51:40 +0530 Subject: [ffmpeg] Add `aac_adtstoasc` when merging if needed Related: #1039 --- yt_dlp/postprocessor/ffmpeg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 25488e58b..6f274b196 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -732,7 +732,9 @@ class FFmpegMergerPP(FFmpegPostProcessor): args = ['-c', 'copy'] for (i, fmt) in enumerate(info['requested_formats']): if fmt.get('acodec') != 'none': - args.extend(['-map', '%u:a:0' % (i)]) + args.extend(['-map', f'{i}:a:0']) + if self.get_audio_codec(fmt['filepath']) == 'aac': + args.extend([f'-bsf:{i}:a:0', 'aac_adtstoasc']) if fmt.get('vcodec') != 'none': args.extend(['-map', '%u:v:0' % (i)]) self.to_screen('Merging formats into "%s"' % filename) -- cgit v1.2.3 From 50eff38c1c071e2d389799843530c294d31887ed Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 23 Sep 2021 11:48:49 +0530 Subject: bugfix for a21e0ab1a1a03f82517cd8cec4b9a2b4d6b81ac3 Closes #1061 --- yt_dlp/postprocessor/ffmpeg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 6f274b196..7ea01620e 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -730,11 +730,13 @@ class FFmpegMergerPP(FFmpegPostProcessor): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') args = ['-c', 'copy'] + audio_streams = 0 for (i, fmt) in enumerate(info['requested_formats']): if fmt.get('acodec') != 'none': args.extend(['-map', f'{i}:a:0']) if self.get_audio_codec(fmt['filepath']) == 'aac': - args.extend([f'-bsf:{i}:a:0', 'aac_adtstoasc']) + args.extend([f'-bsf:a:{audio_streams}', 'aac_adtstoasc']) + audio_streams += 1 if fmt.get('vcodec') != 'none': args.extend(['-map', '%u:v:0' % (i)]) self.to_screen('Merging formats into "%s"' % filename) -- cgit v1.2.3 From b19404591a8ad4d0c7e962931ea809221e3f0b8e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 24 Sep 2021 05:51:54 +0530 Subject: Separate the options `--ignore-errors` and `--no-abort-on-error` In youtube-dl, `-i` ignores both download and post-processing error, and treats the download as successful even if the post-processor fails. yt-dlp used to skip the entire video on either error and there was no option to ignore the post-processing errors like youtube-dl does. By splitting the option into two, now either just the download errors (--no-abort-on-error, default on CLI) or all errors (--ignore-errors) can be ignored as per the users' needs Closes #893 --- yt_dlp/postprocessor/ffmpeg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 7ea01620e..679377aa6 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -288,8 +288,7 @@ class FFmpegPostProcessor(PostProcessor): stdout, stderr = process_communicate_or_kill(p) if p.returncode not in variadic(expected_retcodes): stderr = stderr.decode('utf-8', 'replace').strip() - if self.get_param('verbose', False): - self.report_error(stderr) + self.write_debug(stderr) raise FFmpegPostProcessorError(stderr.split('\n')[-1]) for out_path, _ in output_path_opts: if out_path: -- cgit v1.2.3 From 250a938de82fb6b023c09ce3d89471c5871ff830 Mon Sep 17 00:00:00 2001 From: shirt <2660574+shirt-dev@users.noreply.github.com> Date: Mon, 27 Sep 2021 18:42:33 -0400 Subject: [ffmpeg] Set max probesize to workaround AAC HLS stream issues (#1109) Fixes: #618, #998, #1039 Authored by: shirt-dev --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 679377aa6..ad330ab8e 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -262,7 +262,7 @@ class FFmpegPostProcessor(PostProcessor): oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts if path) - cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] + cmd = [encodeFilename(self.executable, True), encodeArgument('-y'), encodeArgument('-probesize'), encodeArgument('max')] # avconv does not have repeat option if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] -- cgit v1.2.3 From 1f2a268bd33339a5375bffb77a27871213261a13 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 30 Sep 2021 02:15:33 +0530 Subject: [embedsubtitle] Fix error when duration is unknown --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index ad330ab8e..058926929 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -545,7 +545,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): return [], information filename = information['filepath'] - if self._duration_mismatch( + if information.get('duration') and self._duration_mismatch( self._get_real_video_duration(information, False), information['duration']): self.to_screen(f'Skipping {self.pp_key()} since the real and expected durations mismatch') return [], information -- cgit v1.2.3 From e6f21b3d925ea708955c60c400a31fc2e0e36ac0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 17 Sep 2021 23:53:55 +0530 Subject: [docs,cleanup] Some minor refactoring and improve docs --- yt_dlp/postprocessor/ffmpeg.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 058926929..311170920 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -478,7 +478,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegVideoConvertorPP(FFmpegPostProcessor): SUPPORTED_EXTS = ('mp4', 'mkv', 'flv', 'webm', 'mov', 'avi', 'mp3', 'mka', 'm4a', 'ogg', 'opus') FORMAT_RE = re.compile(r'{0}(?:/{0})*$'.format(r'(?:\w+>)?(?:%s)' % '|'.join(SUPPORTED_EXTS))) - _action = 'converting' + _ACTION = 'converting' def __init__(self, downloader=None, preferedformat=None): super(FFmpegVideoConvertorPP, self).__init__(downloader) @@ -497,29 +497,28 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): return [] @PostProcessor._restrict_to(images=False) - def run(self, information): - path, source_ext = information['filepath'], information['ext'].lower() + def run(self, info): + filename, source_ext = info['filepath'], info['ext'].lower() target_ext = self._target_ext(source_ext) _skip_msg = ( - 'could not find a mapping for %s' if not target_ext - else 'already is in target format %s' if source_ext == target_ext + f'could not find a mapping for {source_ext}' if not target_ext + else f'already is in target format {source_ext}' if source_ext == target_ext else None) if _skip_msg: - self.to_screen('Not %s media file "%s"; %s' % (self._action, path, _skip_msg % source_ext)) - return [], information + self.to_screen(f'Not {self._ACTION} media file {filename!r}; {_skip_msg}') + return [], info - prefix, sep, oldext = path.rpartition('.') - outpath = prefix + sep + target_ext - self.to_screen('%s video from %s to %s; Destination: %s' % (self._action.title(), source_ext, target_ext, outpath)) - self.run_ffmpeg(path, outpath, self._options(target_ext)) + outpath = replace_extension(filename, target_ext, source_ext) + self.to_screen(f'{self._ACTION.title()} video from {source_ext} to {target_ext}; Destination: {outpath}') + self.run_ffmpeg(filename, outpath, self._options(target_ext)) - information['filepath'] = outpath - information['format'] = information['ext'] = target_ext - return [path], information + info['filepath'] = outpath + info['format'] = info['ext'] = target_ext + return [filename], info class FFmpegVideoRemuxerPP(FFmpegVideoConvertorPP): - _action = 'remuxing' + _ACTION = 'remuxing' @staticmethod def _options(target_ext): -- cgit v1.2.3 From a1c3967307053767d8c44a5814c88610fe6c4860 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 2 Oct 2021 22:36:31 +0530 Subject: [EmbedSubtitle, SubtitlesConvertor] Fix error when subtitle file is missing Closes #1152, #1134 Bug from 8e25d624df003d691be922488d6ab7007f75333d --- yt_dlp/postprocessor/ffmpeg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 311170920..6bb66569a 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -555,7 +555,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): mp4_ass_warn = False for lang, sub_info in subtitles.items(): - if not os.path.exists(information.get('filepath', '')): + if not os.path.exists(sub_info.get('filepath', '')): self.report_warning(f'Skipping embedding {lang} subtitle because the file is missing') continue sub_ext = sub_info['ext'] @@ -845,6 +845,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): self.to_screen('Converting subtitles') sub_filenames = [] for lang, sub in subs.items(): + if not os.path.exists(sub.get('filepath', '')): + self.report_warning(f'Skipping embedding {lang} subtitle because the file is missing') + continue ext = sub['ext'] if ext == new_ext: self.to_screen('Subtitle file for %s is already in the requested format' % new_ext) -- cgit v1.2.3 From 91b6c884c9c02a2a8ffe247131d05e8e8a6021a4 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 10 Oct 2021 02:56:30 +0530 Subject: Revert "[ffmpeg] Set max probesize to workaround AAC HLS stream issues (#1109)" This reverts commit 250a938de82fb6b023c09ce3d89471c5871ff830. This is no longer necessary since 7687c8ac6e223a725b3ef8f56f04779bebdc86c5 --- yt_dlp/postprocessor/ffmpeg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 6bb66569a..5f6861f93 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -262,7 +262,7 @@ class FFmpegPostProcessor(PostProcessor): oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts if path) - cmd = [encodeFilename(self.executable, True), encodeArgument('-y'), encodeArgument('-probesize'), encodeArgument('max')] + cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] # avconv does not have repeat option if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] -- cgit v1.2.3 From 9dda99f2fca7342c8f19150ac8730d67fceed42d Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Oct 2021 15:27:00 +0530 Subject: [Merger] Do not add `aac_adtstoasc` to non-hls audio --- yt_dlp/postprocessor/ffmpeg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 5f6861f93..e6aa2940a 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -732,7 +732,8 @@ class FFmpegMergerPP(FFmpegPostProcessor): for (i, fmt) in enumerate(info['requested_formats']): if fmt.get('acodec') != 'none': args.extend(['-map', f'{i}:a:0']) - if self.get_audio_codec(fmt['filepath']) == 'aac': + aac_fixup = fmt['protocol'].startswith('m3u8') and self.get_audio_codec(fmt['filepath']) == 'aac' + if aac_fixup: args.extend([f'-bsf:a:{audio_streams}', 'aac_adtstoasc']) audio_streams += 1 if fmt.get('vcodec') != 'none': -- cgit v1.2.3 From b11d210156f083f23e1bce284192314e54e4047a Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Oct 2021 09:19:25 +0530 Subject: [EmbedMetadata] Allow overwriting all default metadata with `meta_default` key --- yt_dlp/postprocessor/ffmpeg.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'yt_dlp/postprocessor/ffmpeg.py') diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index e6aa2940a..e5595341d 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -10,7 +10,7 @@ import json from .common import AudioConversionError, PostProcessor -from ..compat import compat_str, compat_numeric_types +from ..compat import compat_str from ..utils import ( dfxp2srt, encodeArgument, @@ -664,15 +664,14 @@ class FFmpegMetadataPP(FFmpegPostProcessor): def _get_metadata_opts(self, info): metadata = {} + meta_prefix = 'meta_' def add(meta_list, info_list=None): - if not meta_list: - return - for info_f in variadic(info_list or meta_list): - if isinstance(info.get(info_f), (compat_str, compat_numeric_types)): - for meta_f in variadic(meta_list): - metadata[meta_f] = info[info_f] - break + value = next(( + str(info[key]) for key in [meta_prefix] + list(variadic(info_list or meta_list)) + if info.get(key) is not None), None) + if value not in ('', None): + metadata.update({meta_f: value for meta_f in variadic(meta_list)}) # See [1-4] for some info on media metadata/metadata supported # by ffmpeg. @@ -695,9 +694,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('episode_id', ('episode', 'episode_id')) add('episode_sort', 'episode_number') - prefix = 'meta_' - for key in filter(lambda k: k.startswith(prefix), info.keys()): - add(key[len(prefix):], key) + for key, value in info.items(): + if value is not None and key != meta_prefix and key.startswith(meta_prefix): + metadata[key[len(meta_prefix):]] = value for name, value in metadata.items(): yield ('-metadata', f'{name}={value}') -- cgit v1.2.3