diff options
author | Jesús <heckyel@hyperbola.info> | 2021-10-18 15:24:21 -0500 |
---|---|---|
committer | Jesús <heckyel@hyperbola.info> | 2021-10-18 15:24:21 -0500 |
commit | 5122028a4bcac4ae577ef7fbd55ccad5cb34ef5e (patch) | |
tree | 65209bc739db35e31f1c9b5b868eb5df4fe12ae3 /hypervideo_dl/postprocessor | |
parent | 27fe903c511691c078942bef5ee9a05a43b15c8f (diff) | |
download | hypervideo-5122028a4bcac4ae577ef7fbd55ccad5cb34ef5e.tar.lz hypervideo-5122028a4bcac4ae577ef7fbd55ccad5cb34ef5e.tar.xz hypervideo-5122028a4bcac4ae577ef7fbd55ccad5cb34ef5e.zip |
update from upstream
Diffstat (limited to 'hypervideo_dl/postprocessor')
-rw-r--r-- | hypervideo_dl/postprocessor/__init__.py | 43 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/common.py | 125 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/embedthumbnail.py | 279 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/exec.py | 42 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/ffmpeg.py | 732 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/metadataparser.py | 116 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/modify_chapters.py | 336 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/movefilesafterdownload.py | 54 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/sponskrub.py | 96 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/sponsorblock.py | 96 | ||||
-rw-r--r-- | hypervideo_dl/postprocessor/xattrpp.py | 13 |
11 files changed, 1624 insertions, 308 deletions
diff --git a/hypervideo_dl/postprocessor/__init__.py b/hypervideo_dl/postprocessor/__init__.py index 3ea5183..07c87b7 100644 --- a/hypervideo_dl/postprocessor/__init__.py +++ b/hypervideo_dl/postprocessor/__init__.py @@ -1,40 +1,43 @@ -from __future__ import unicode_literals +# flake8: noqa: F401 + +from ..utils import load_plugins from .embedthumbnail import EmbedThumbnailPP +from .exec import ExecPP, ExecAfterDownloadPP from .ffmpeg import ( FFmpegPostProcessor, FFmpegEmbedSubtitlePP, FFmpegExtractAudioPP, + FFmpegFixupDurationPP, FFmpegFixupStretchedPP, + FFmpegFixupTimestampPP, FFmpegFixupM3u8PP, FFmpegFixupM4aPP, FFmpegMergerPP, FFmpegMetadataPP, - FFmpegVideoConvertorPP, FFmpegSubtitlesConvertorPP, + FFmpegThumbnailsConvertorPP, + FFmpegSplitChaptersPP, + FFmpegVideoConvertorPP, + FFmpegVideoRemuxerPP, ) +from .metadataparser import ( + MetadataFromFieldPP, + MetadataFromTitlePP, + MetadataParserPP, +) +from .modify_chapters import ModifyChaptersPP +from .movefilesafterdownload import MoveFilesAfterDownloadPP +from .sponskrub import SponSkrubPP +from .sponsorblock import SponsorBlockPP from .xattrpp import XAttrMetadataPP -from .execafterdownload import ExecAfterDownloadPP -from .metadatafromtitle import MetadataFromTitlePP + +_PLUGIN_CLASSES = load_plugins('postprocessor', 'PP', globals()) def get_postprocessor(key): return globals()[key + 'PP'] -__all__ = [ - 'EmbedThumbnailPP', - 'ExecAfterDownloadPP', - 'FFmpegEmbedSubtitlePP', - 'FFmpegExtractAudioPP', - 'FFmpegFixupM3u8PP', - 'FFmpegFixupM4aPP', - 'FFmpegFixupStretchedPP', - 'FFmpegMergerPP', - 'FFmpegMetadataPP', - 'FFmpegPostProcessor', - 'FFmpegSubtitlesConvertorPP', - 'FFmpegVideoConvertorPP', - 'MetadataFromTitlePP', - 'XAttrMetadataPP', -] +__all__ = [name for name in globals().keys() if name.endswith('IE')] +__all__.append('FFmpegPostProcessor') diff --git a/hypervideo_dl/postprocessor/common.py b/hypervideo_dl/postprocessor/common.py index 599dd1d..b491afb 100644 --- a/hypervideo_dl/postprocessor/common.py +++ b/hypervideo_dl/postprocessor/common.py @@ -1,15 +1,38 @@ from __future__ import unicode_literals +import copy +import functools import os +from ..compat import compat_str from ..utils import ( - PostProcessingError, - cli_configuration_args, + _configuration_args, encodeFilename, + PostProcessingError, ) -class PostProcessor(object): +class PostProcessorMetaClass(type): + @staticmethod + def run_wrapper(func): + @functools.wraps(func) + def run(self, info, *args, **kwargs): + info_copy = copy.deepcopy(self._copy_infodict(info)) + self._hook_progress({'status': 'started'}, info_copy) + ret = func(self, info, *args, **kwargs) + if ret is not None: + _, info = ret + self._hook_progress({'status': 'finished'}, info_copy) + return ret + return run + + def __new__(cls, name, bases, attrs): + if 'run' in attrs: + attrs['run'] = cls.run_wrapper(attrs['run']) + return type.__new__(cls, name, bases, attrs) + + +class PostProcessor(metaclass=PostProcessorMetaClass): """Post Processor class. PostProcessor objects can be added to downloaders with their @@ -32,11 +55,66 @@ class PostProcessor(object): _downloader = None def __init__(self, downloader=None): - self._downloader = downloader + self._progress_hooks = [] + self.add_progress_hook(self.report_progress) + self.set_downloader(downloader) + self.PP_NAME = self.pp_key() + + @classmethod + def pp_key(cls): + name = cls.__name__[:-2] + return compat_str(name[6:]) if name[:6].lower() == 'ffmpeg' else name + + def to_screen(self, text, prefix=True, *args, **kwargs): + tag = '[%s] ' % self.PP_NAME if prefix else '' + if self._downloader: + return self._downloader.to_screen('%s%s' % (tag, text), *args, **kwargs) + + def report_warning(self, text, *args, **kwargs): + if self._downloader: + return self._downloader.report_warning(text, *args, **kwargs) + + def report_error(self, text, *args, **kwargs): + # Exists only for compatibility. Do not use + if self._downloader: + return self._downloader.report_error(text, *args, **kwargs) + + def write_debug(self, text, *args, **kwargs): + if self._downloader: + return self._downloader.write_debug(text, *args, **kwargs) + + def get_param(self, name, default=None, *args, **kwargs): + if self._downloader: + return self._downloader.params.get(name, default, *args, **kwargs) + return default def set_downloader(self, downloader): """Sets the downloader for this PP.""" self._downloader = downloader + for ph in getattr(downloader, '_postprocessor_hooks', []): + self.add_progress_hook(ph) + + def _copy_infodict(self, info_dict): + return getattr(self._downloader, '_copy_infodict', dict)(info_dict) + + @staticmethod + def _restrict_to(*, video=True, audio=True, images=True): + allowed = {'video': video, 'audio': audio, 'images': images} + + def decorator(func): + @functools.wraps(func) + def wrapper(self, info): + format_type = ( + 'video' if info.get('vcodec') != 'none' + else 'audio' if info.get('acodec') != 'none' + else 'images') + if allowed[format_type]: + return func(self, info) + else: + self.to_screen('Skipping %s' % format_type) + return [], info + return wrapper + return decorator def run(self, information): """Run the PostProcessor. @@ -59,10 +137,41 @@ class PostProcessor(object): try: os.utime(encodeFilename(path), (atime, mtime)) except Exception: - self._downloader.report_warning(errnote) - - def _configuration_args(self, default=[]): - return cli_configuration_args(self._downloader.params, 'postprocessor_args', default) + self.report_warning(errnote) + + def _configuration_args(self, exe, *args, **kwargs): + return _configuration_args( + self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs) + + def _hook_progress(self, status, info_dict): + if not self._progress_hooks: + return + status.update({ + 'info_dict': info_dict, + 'postprocessor': self.pp_key(), + }) + for ph in self._progress_hooks: + ph(status) + + def add_progress_hook(self, ph): + # See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface + self._progress_hooks.append(ph) + + def report_progress(self, s): + s['_default_template'] = '%(postprocessor)s %(status)s' % s + + progress_dict = s.copy() + progress_dict.pop('info_dict') + progress_dict = {'info': s['info_dict'], 'progress': progress_dict} + + progress_template = self.get_param('progress_template', {}) + tmpl = progress_template.get('postprocess') + if tmpl: + self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict)) + + self._downloader.to_console_title(self._downloader.evaluate_outtmpl( + progress_template.get('postprocess-title') or 'hypervideo %(progress._default_template)s', + progress_dict)) class AudioConversionError(PostProcessingError): diff --git a/hypervideo_dl/postprocessor/embedthumbnail.py b/hypervideo_dl/postprocessor/embedthumbnail.py index 3990908..3139a63 100644 --- a/hypervideo_dl/postprocessor/embedthumbnail.py +++ b/hypervideo_dl/postprocessor/embedthumbnail.py @@ -1,20 +1,35 @@ # coding: utf-8 from __future__ import unicode_literals - +import base64 +import imghdr import os import subprocess - -from .ffmpeg import FFmpegPostProcessor - +import re + +try: + from mutagen.flac import Picture, FLAC + from mutagen.mp4 import MP4, MP4Cover + from mutagen.oggopus import OggOpus + from mutagen.oggvorbis import OggVorbis + has_mutagen = True +except ImportError: + has_mutagen = False + +from .common import PostProcessor +from .ffmpeg import ( + FFmpegPostProcessor, + FFmpegThumbnailsConvertorPP, +) from ..utils import ( check_executable, encodeArgument, encodeFilename, + error_to_compat_str, PostProcessingError, prepend_extension, - replace_extension, - shell_quote + process_communicate_or_kill, + shell_quote, ) @@ -23,108 +38,198 @@ class EmbedThumbnailPPError(PostProcessingError): class EmbedThumbnailPP(FFmpegPostProcessor): + def __init__(self, downloader=None, already_have_thumbnail=False): - super(EmbedThumbnailPP, self).__init__(downloader) + FFmpegPostProcessor.__init__(self, downloader) self._already_have_thumbnail = already_have_thumbnail + def _get_thumbnail_resolution(self, filename, thumbnail_dict): + def guess(): + width, height = thumbnail_dict.get('width'), thumbnail_dict.get('height') + if width and height: + return width, height + + try: + size_regex = r',\s*(?P<w>\d+)x(?P<h>\d+)\s*[,\[]' + size_result = self.run_ffmpeg(filename, None, ['-hide_banner'], expected_retcodes=(1,)) + mobj = re.search(size_regex, size_result) + if mobj is None: + return guess() + except PostProcessingError as err: + self.report_warning('unable to find the thumbnail resolution; %s' % error_to_compat_str(err)) + return guess() + return int(mobj.group('w')), int(mobj.group('h')) + + def _report_run(self, exe, filename): + self.to_screen('%s: Adding thumbnail to "%s"' % (exe, filename)) + + @PostProcessor._restrict_to(images=False) def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') if not info.get('thumbnails'): - self._downloader.to_screen('[embedthumbnail] There aren\'t any thumbnails to embed') + self.to_screen('There aren\'t any thumbnails to embed') return [], info - thumbnail_filename = info['thumbnails'][-1]['filename'] - + idx = next((-i for i, t in enumerate(info['thumbnails'][::-1], 1) if t.get('filepath')), None) + if idx is None: + self.to_screen('There are no thumbnails on disk') + return [], info + thumbnail_filename = info['thumbnails'][idx]['filepath'] if not os.path.exists(encodeFilename(thumbnail_filename)): - self._downloader.report_warning( - 'Skipping embedding the thumbnail because the file is missing.') + self.report_warning('Skipping embedding the thumbnail because the file is missing.') return [], info - def is_webp(path): - with open(encodeFilename(path), 'rb') as f: - b = f.read(12) - return b[0:4] == b'RIFF' and b[8:] == b'WEBP' - # Correct extension for WebP file with wrong extension (see #25687, #25717) - _, thumbnail_ext = os.path.splitext(thumbnail_filename) - if thumbnail_ext: - thumbnail_ext = thumbnail_ext[1:].lower() - if thumbnail_ext != 'webp' and is_webp(thumbnail_filename): - self._downloader.to_screen( - '[ffmpeg] Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename) - thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp') - os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename)) - thumbnail_filename = thumbnail_webp_filename - thumbnail_ext = 'webp' - - # Convert unsupported thumbnail formats to JPEG (see #25687, #25717) - if thumbnail_ext not in ['jpg', 'png']: - # NB: % is supposed to be escaped with %% but this does not work - # for input files so working around with standard substitution - escaped_thumbnail_filename = thumbnail_filename.replace('%', '#') - os.rename(encodeFilename(thumbnail_filename), encodeFilename(escaped_thumbnail_filename)) - escaped_thumbnail_jpg_filename = replace_extension(escaped_thumbnail_filename, 'jpg') - self._downloader.to_screen('[ffmpeg] Converting thumbnail "%s" to JPEG' % escaped_thumbnail_filename) - self.run_ffmpeg(escaped_thumbnail_filename, escaped_thumbnail_jpg_filename, ['-bsf:v', 'mjpeg2jpeg']) - os.remove(encodeFilename(escaped_thumbnail_filename)) - thumbnail_jpg_filename = replace_extension(thumbnail_filename, 'jpg') - # Rename back to unescaped for further processing - os.rename(encodeFilename(escaped_thumbnail_jpg_filename), encodeFilename(thumbnail_jpg_filename)) - thumbnail_filename = thumbnail_jpg_filename - - if info['ext'] == 'mp3': - options = [ - '-c', 'copy', '-map', '0', '-map', '1', - '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (Front)"'] + convertor = FFmpegThumbnailsConvertorPP(self._downloader) + convertor.fixup_webp(info, idx) - self._downloader.to_screen('[ffmpeg] Adding thumbnail to "%s"' % filename) + original_thumbnail = thumbnail_filename = info['thumbnails'][idx]['filepath'] - self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options) - - if not self._already_have_thumbnail: - os.remove(encodeFilename(thumbnail_filename)) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) - - elif info['ext'] in ['m4a', 'mp4']: - atomicparsley = next((x - for x in ['AtomicParsley', 'atomicparsley'] - if check_executable(x, ['-v'])), None) + # Convert unsupported thumbnail formats to PNG (see #25687, #25717) + # Original behavior was to convert to JPG, but since JPG is a lossy + # format, there will be some additional data loss. + # PNG, on the other hand, is lossless. + thumbnail_ext = os.path.splitext(thumbnail_filename)[1][1:] + if thumbnail_ext not in ('jpg', 'jpeg', 'png'): + thumbnail_filename = convertor.convert_thumbnail(thumbnail_filename, 'png') + thumbnail_ext = 'png' - if atomicparsley is None: - raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.') + mtime = os.stat(encodeFilename(filename)).st_mtime - cmd = [encodeFilename(atomicparsley, True), - encodeFilename(filename, True), - encodeArgument('--artwork'), - encodeFilename(thumbnail_filename, True), - encodeArgument('-o'), - encodeFilename(temp_filename, True)] + success = True + if info['ext'] == 'mp3': + options = [ + '-c', 'copy', '-map', '0:0', '-map', '1:0', '-id3v2_version', '3', + '-metadata:s:v', 'title="Album cover"', '-metadata:s:v', 'comment="Cover (front)"'] - self._downloader.to_screen('[atomicparsley] Adding thumbnail to "%s"' % filename) + self._report_run('ffmpeg', filename) + self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options) - if self._downloader.params.get('verbose', False): - self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd)) + elif info['ext'] in ['mkv', 'mka']: + options = ['-c', 'copy', '-map', '0', '-dn'] + + mimetype = 'image/%s' % ('png' if thumbnail_ext == 'png' else 'jpeg') + old_stream, new_stream = self.get_stream_number( + filename, ('tags', 'mimetype'), mimetype) + if old_stream is not None: + options.extend(['-map', '-0:%d' % old_stream]) + new_stream -= 1 + options.extend([ + '-attach', thumbnail_filename, + '-metadata:s:%d' % new_stream, 'mimetype=%s' % mimetype, + '-metadata:s:%d' % new_stream, 'filename=cover.%s' % thumbnail_ext]) + + self._report_run('ffmpeg', filename) + self.run_ffmpeg(filename, temp_filename, options) + + elif info['ext'] in ['m4a', 'mp4', 'mov']: + prefer_atomicparsley = 'embed-thumbnail-atomicparsley' in self.get_param('compat_opts', []) + # Method 1: Use mutagen + if not has_mutagen or prefer_atomicparsley: + success = False + else: + try: + self._report_run('mutagen', filename) + meta = MP4(filename) + # NOTE: the 'covr' atom is a non-standard MPEG-4 atom, + # Apple iTunes 'M4A' files include the 'moov.udta.meta.ilst' atom. + f = {'jpeg': MP4Cover.FORMAT_JPEG, 'png': MP4Cover.FORMAT_PNG}[imghdr.what(thumbnail_filename)] + with open(thumbnail_filename, 'rb') as thumbfile: + thumb_data = thumbfile.read() + meta.tags['covr'] = [MP4Cover(data=thumb_data, imageformat=f)] + meta.save() + temp_filename = filename + except Exception as err: + self.report_warning('unable to embed using mutagen; %s' % error_to_compat_str(err)) + success = False + + # Method 2: Use ffmpeg+ffprobe + if not success and not prefer_atomicparsley: + success = True + try: + options = ['-c', 'copy', '-map', '0', '-dn', '-map', '1'] + + old_stream, new_stream = self.get_stream_number( + filename, ('disposition', 'attached_pic'), 1) + if old_stream is not None: + options.extend(['-map', '-0:%d' % old_stream]) + new_stream -= 1 + options.extend(['-disposition:%s' % new_stream, 'attached_pic']) + + self._report_run('ffmpeg', filename) + self.run_ffmpeg_multiple_files([filename, thumbnail_filename], temp_filename, options) + except PostProcessingError as err: + self.report_warning('unable to embed using ffprobe & ffmpeg; %s' % error_to_compat_str(err)) + success = False + + # Method 3: Use AtomicParsley + if not success: + success = True + atomicparsley = next(( + x for x in ['AtomicParsley', 'atomicparsley'] + if check_executable(x, ['-v'])), None) + if atomicparsley is None: + raise EmbedThumbnailPPError('AtomicParsley was not found. Please install') + + cmd = [encodeFilename(atomicparsley, True), + encodeFilename(filename, True), + encodeArgument('--artwork'), + encodeFilename(thumbnail_filename, True), + encodeArgument('-o'), + encodeFilename(temp_filename, True)] + cmd += [encodeArgument(o) for o in self._configuration_args('AtomicParsley')] + + self._report_run('atomicparsley', filename) + self.write_debug('AtomicParsley command line: %s' % shell_quote(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process_communicate_or_kill(p) + if p.returncode != 0: + msg = stderr.decode('utf-8', 'replace').strip() + raise EmbedThumbnailPPError(msg) + # for formats that don't support thumbnails (like 3gp) AtomicParsley + # won't create to the temporary file + if b'No changes' in stdout: + self.report_warning('The file format doesn\'t support embedding a thumbnail') + success = False + + elif info['ext'] in ['ogg', 'opus', 'flac']: + if not has_mutagen: + raise EmbedThumbnailPPError('module mutagen was not found. Please install using `python -m pip install mutagen`') + + self._report_run('mutagen', filename) + f = {'opus': OggOpus, 'flac': FLAC, 'ogg': OggVorbis}[info['ext']](filename) + + pic = Picture() + pic.mime = 'image/%s' % imghdr.what(thumbnail_filename) + with open(thumbnail_filename, 'rb') as thumbfile: + pic.data = thumbfile.read() + pic.type = 3 # front cover + res = self._get_thumbnail_resolution(thumbnail_filename, info['thumbnails'][idx]) + if res is not None: + pic.width, pic.height = res + + if info['ext'] == 'flac': + f.add_picture(pic) + else: + # https://wiki.xiph.org/VorbisComment#METADATA_BLOCK_PICTURE + f['METADATA_BLOCK_PICTURE'] = base64.b64encode(pic.write()).decode('ascii') + f.save() + temp_filename = filename - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + else: + raise EmbedThumbnailPPError('Supported filetypes for thumbnail embedding are: mp3, mkv/mka, ogg/opus/flac, m4a/mp4/mov') - if p.returncode != 0: - msg = stderr.decode('utf-8', 'replace').strip() - raise EmbedThumbnailPPError(msg) + if success and temp_filename != filename: + os.replace(temp_filename, filename) - if not self._already_have_thumbnail: - os.remove(encodeFilename(thumbnail_filename)) - # for formats that don't support thumbnails (like 3gp) AtomicParsley - # won't create to the temporary file - if b'No changes' in stdout: - self._downloader.report_warning('The file format doesn\'t support embedding a thumbnail') - else: - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) - else: - raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.') + self.try_utime(filename, mtime, mtime) - return [], info + files_to_delete = [thumbnail_filename] + if self._already_have_thumbnail: + if original_thumbnail == thumbnail_filename: + files_to_delete = [] + elif original_thumbnail != thumbnail_filename: + files_to_delete.append(original_thumbnail) + return files_to_delete, info diff --git a/hypervideo_dl/postprocessor/exec.py b/hypervideo_dl/postprocessor/exec.py new file mode 100644 index 0000000..7a3cb49 --- /dev/null +++ b/hypervideo_dl/postprocessor/exec.py @@ -0,0 +1,42 @@ +from __future__ import unicode_literals + +import subprocess + +from .common import PostProcessor +from ..compat import compat_shlex_quote +from ..utils import ( + encodeArgument, + PostProcessingError, + variadic, +) + + +class ExecPP(PostProcessor): + + def __init__(self, downloader, exec_cmd): + PostProcessor.__init__(self, downloader) + self.exec_cmd = variadic(exec_cmd) + + def parse_cmd(self, cmd, info): + tmpl, tmpl_dict = self._downloader.prepare_outtmpl(cmd, info) + if tmpl_dict: # if there are no replacements, tmpl_dict = {} + return self._downloader.escape_outtmpl(tmpl) % tmpl_dict + + # If no replacements are found, replace {} for backard compatibility + if '{}' not in cmd: + cmd += ' {}' + return cmd.replace('{}', compat_shlex_quote( + info.get('filepath') or info['_filename'])) + + def run(self, info): + for tmpl in self.exec_cmd: + cmd = self.parse_cmd(tmpl, info) + self.to_screen('Executing command: %s' % cmd) + retCode = subprocess.call(encodeArgument(cmd), shell=True) + if retCode != 0: + raise PostProcessingError('Command returned error code %d' % retCode) + return [], info + + +class ExecAfterDownloadPP(ExecPP): # for backward compatibility + pass diff --git a/hypervideo_dl/postprocessor/ffmpeg.py b/hypervideo_dl/postprocessor/ffmpeg.py index 3078329..a6d6d78 100644 --- a/hypervideo_dl/postprocessor/ffmpeg.py +++ b/hypervideo_dl/postprocessor/ffmpeg.py @@ -1,26 +1,32 @@ from __future__ import unicode_literals import io +import itertools import os import subprocess import time import re - +import json from .common import AudioConversionError, PostProcessor +from ..compat import compat_str from ..utils import ( + dfxp2srt, encodeArgument, encodeFilename, + float_or_none, get_exe_version, is_outdated_version, + ISO639Utils, + orderedSet, PostProcessingError, prepend_extension, - shell_quote, - subtitles_filename, - dfxp2srt, - ISO639Utils, + process_communicate_or_kill, replace_extension, + shell_quote, + traverse_obj, + variadic, ) @@ -58,15 +64,14 @@ class FFmpegPostProcessor(PostProcessor): def check_version(self): if not self.available: - raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.') + raise FFmpegPostProcessorError('ffmpeg not found. Please install or provide the path using --ffmpeg-location') required_version = '10-0' if self.basename == 'avconv' else '1.0' if is_outdated_version( self._versions[self.basename], required_version): warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % ( self.basename, self.basename, required_version) - if self._downloader: - self._downloader.report_warning(warning) + self.report_warning(warning) @staticmethod def get_versions(downloader=None): @@ -96,30 +101,28 @@ class FFmpegPostProcessor(PostProcessor): self._paths = None self._versions = None if self._downloader: - prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True) - location = self._downloader.params.get('ffmpeg_location') + prefer_ffmpeg = self.get_param('prefer_ffmpeg', True) + location = self.get_param('ffmpeg_location') if location is not None: if not os.path.exists(location): - self._downloader.report_warning( + self.report_warning( 'ffmpeg-location %s does not exist! ' - 'Continuing without avconv/ffmpeg.' % (location)) + 'Continuing without ffmpeg.' % (location)) self._versions = {} return - elif not os.path.isdir(location): + elif os.path.isdir(location): + dirname, basename = location, None + else: basename = os.path.splitext(os.path.basename(location))[0] - if basename not in programs: - self._downloader.report_warning( - 'Cannot identify executable %s, its basename should be one of %s. ' - 'Continuing without avconv/ffmpeg.' % - (location, ', '.join(programs))) - self._versions = {} - return None - location = os.path.dirname(os.path.abspath(location)) + basename = next((p for p in programs if basename.startswith(p)), 'ffmpeg') + dirname = os.path.dirname(os.path.abspath(location)) if basename in ('ffmpeg', 'ffprobe'): prefer_ffmpeg = True self._paths = dict( - (p, os.path.join(location, p)) for p in programs) + (p, os.path.join(dirname, p)) for p in programs) + if basename: + self._paths[basename] = location self._versions = dict( (p, get_ffmpeg_version(self._paths[p])) for p in programs) if self._versions is None: @@ -163,7 +166,7 @@ class FFmpegPostProcessor(PostProcessor): def get_audio_codec(self, path): if not self.probe_available and not self.available: - raise PostProcessingError('ffprobe/avprobe and ffmpeg/avconv not found. Please install one.') + raise PostProcessingError('ffprobe and ffmpeg not found. Please install or provide the path using --ffmpeg-location') try: if self.probe_available: cmd = [ @@ -174,13 +177,11 @@ class FFmpegPostProcessor(PostProcessor): encodeFilename(self.executable, True), encodeArgument('-i')] cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True)) - if self._downloader.params.get('verbose', False): - self._downloader.to_screen( - '[debug] %s command line: %s' % (self.basename, shell_quote(cmd))) + self.write_debug('%s command line: %s' % (self.basename, shell_quote(cmd))) handle = subprocess.Popen( cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - stdout_data, stderr_data = handle.communicate() + stdout_data, stderr_data = process_communicate_or_kill(handle) expected_ret = 0 if self.probe_available else 1 if handle.wait() != expected_ret: return None @@ -203,55 +204,174 @@ class FFmpegPostProcessor(PostProcessor): return mobj.group(1) return None - def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + def get_metadata_object(self, path, opts=[]): + if self.probe_basename != 'ffprobe': + if self.probe_available: + self.report_warning('Only ffprobe is supported for metadata extraction') + raise PostProcessingError('ffprobe not found. Please install or provide the path using --ffmpeg-location') self.check_version() - oldest_mtime = min( - os.stat(encodeFilename(path)).st_mtime for path in input_paths) + cmd = [ + encodeFilename(self.probe_executable, True), + encodeArgument('-hide_banner'), + encodeArgument('-show_format'), + encodeArgument('-show_streams'), + encodeArgument('-print_format'), + encodeArgument('json'), + ] + + cmd += opts + cmd.append(encodeFilename(self._ffmpeg_filename_argument(path), True)) + self.write_debug('ffprobe command line: %s' % shell_quote(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, stderr = p.communicate() + return json.loads(stdout.decode('utf-8', 'replace')) + + def get_stream_number(self, path, keys, value): + streams = self.get_metadata_object(path)['streams'] + num = next( + (i for i, stream in enumerate(streams) if traverse_obj(stream, keys, casesense=False) == value), + None) + return num, len(streams) - opts += self._configuration_args() + def _get_real_video_duration(self, info, fatal=True): + try: + if '_real_duration' not in info: + info['_real_duration'] = float_or_none( + traverse_obj(self.get_metadata_object(info['filepath']), ('format', 'duration'))) + if not info['_real_duration']: + raise PostProcessingError('ffprobe returned empty duration') + except PostProcessingError as e: + if fatal: + raise PostProcessingError(f'Unable to determine video duration; {e}') + return info.setdefault('_real_duration', None) + + def _duration_mismatch(self, d1, d2): + if not d1 or not d2: + return None + return abs(d1 - d2) > 1 + + def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, **kwargs): + return self.real_run_ffmpeg( + [(path, []) for path in input_paths], + [(out_path, opts)], **kwargs) + + def real_run_ffmpeg(self, input_path_opts, output_path_opts, *, expected_retcodes=(0,)): + self.check_version() + + oldest_mtime = min( + os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts if path) - files_cmd = [] - for path in input_paths: - files_cmd.extend([ - encodeArgument('-i'), - encodeFilename(self._ffmpeg_filename_argument(path), True) - ]) cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] # avconv does not have repeat option if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] - cmd += (files_cmd - + [encodeArgument(o) for o in opts] - + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) - if self._downloader.params.get('verbose', False): - self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd)) + def make_args(file, args, name, number): + keys = ['_%s%d' % (name, number), '_%s' % name] + if name == 'o' and number == 1: + keys.append('') + args += self._configuration_args(self.basename, keys) + if name == 'i': + args.append('-i') + return ( + [encodeArgument(arg) for arg in args] + + [encodeFilename(self._ffmpeg_filename_argument(file), True)]) + + for arg_type, path_opts in (('i', input_path_opts), ('o', output_path_opts)): + cmd += itertools.chain.from_iterable( + make_args(path, list(opts), arg_type, i + 1) + for i, (path, opts) in enumerate(path_opts) if path) + + self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) - stdout, stderr = p.communicate() - if p.returncode != 0: - stderr = stderr.decode('utf-8', 'replace') - msg = stderr.strip().split('\n')[-1] - raise FFmpegPostProcessorError(msg) - self.try_utime(out_path, oldest_mtime, oldest_mtime) + stdout, stderr = process_communicate_or_kill(p) + if p.returncode not in variadic(expected_retcodes): + stderr = stderr.decode('utf-8', 'replace').strip() + self.write_debug(stderr) + raise FFmpegPostProcessorError(stderr.split('\n')[-1]) + for out_path, _ in output_path_opts: + if out_path: + self.try_utime(out_path, oldest_mtime, oldest_mtime) + return stderr.decode('utf-8', 'replace') + + def run_ffmpeg(self, path, out_path, opts, **kwargs): + return self.run_ffmpeg_multiple_files([path], out_path, opts, **kwargs) - def run_ffmpeg(self, path, out_path, opts): - self.run_ffmpeg_multiple_files([path], out_path, opts) - - def _ffmpeg_filename_argument(self, fn): + @staticmethod + def _ffmpeg_filename_argument(fn): # Always use 'file:' because the filename may contain ':' (ffmpeg # interprets that as a protocol) or can start with '-' (-- is broken in # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details) # Also leave '-' intact in order not to break streaming to stdout. + if fn.startswith(('http://', 'https://')): + return fn return 'file:' + fn if fn != '-' else fn + @staticmethod + def _quote_for_ffmpeg(string): + # See https://ffmpeg.org/ffmpeg-utils.html#toc-Quoting-and-escaping + # A sequence of '' produces '\'''\''; + # final replace removes the empty '' between \' \'. + string = string.replace("'", r"'\''").replace("'''", "'") + # Handle potential ' at string boundaries. + string = string[1:] if string[0] == "'" else "'" + string + return string[:-1] if string[-1] == "'" else string + "'" + + def force_keyframes(self, filename, timestamps): + timestamps = orderedSet(timestamps) + if timestamps[0] == 0: + timestamps = timestamps[1:] + keyframe_file = prepend_extension(filename, 'keyframes.temp') + self.to_screen(f'Re-encoding "{filename}" with appropriate keyframes') + self.run_ffmpeg(filename, keyframe_file, ['-force_key_frames', ','.join( + f'{t:.6f}' for t in timestamps)]) + return keyframe_file + + def concat_files(self, in_files, out_file, concat_opts=None): + """ + Use concat demuxer to concatenate multiple files having identical streams. + + Only inpoint, outpoint, and duration concat options are supported. + See https://ffmpeg.org/ffmpeg-formats.html#concat-1 for details + """ + concat_file = f'{out_file}.concat' + self.write_debug(f'Writing concat spec to {concat_file}') + with open(concat_file, 'wt', encoding='utf-8') as f: + f.writelines(self._concat_spec(in_files, concat_opts)) + + out_flags = ['-c', 'copy'] + if out_file.rpartition('.')[-1] in ('mp4', 'mov'): + # For some reason, '-c copy' is not enough to copy subtitles + out_flags.extend(['-c:s', 'mov_text', '-movflags', '+faststart']) + + try: + self.real_run_ffmpeg( + [(concat_file, ['-hide_banner', '-nostdin', '-f', 'concat', '-safe', '0'])], + [(out_file, out_flags)]) + finally: + os.remove(concat_file) + + @classmethod + def _concat_spec(cls, in_files, concat_opts=None): + if concat_opts is None: + concat_opts = [{}] * len(in_files) + yield 'ffconcat version 1.0\n' + for file, opts in zip(in_files, concat_opts): + yield f'file {cls._quote_for_ffmpeg(cls._ffmpeg_filename_argument(file))}\n' + # Iterate explicitly to yield the following directives in order, ignoring the rest. + for directive in 'inpoint', 'outpoint', 'duration': + if directive in opts: + yield f'{directive} {opts[directive]}\n' + class FFmpegExtractAudioPP(FFmpegPostProcessor): + COMMON_AUDIO_EXTS = ('wav', 'flac', 'm4a', 'aiff', 'mp3', 'ogg', 'mka', 'opus', 'wma') + SUPPORTED_EXTS = ('best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav') + def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): FFmpegPostProcessor.__init__(self, downloader) - if preferredcodec is None: - preferredcodec = 'best' - self._preferredcodec = preferredcodec + self._preferredcodec = preferredcodec or 'best' self._preferredquality = preferredquality self._nopostoverwrites = nopostoverwrites @@ -266,8 +386,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): except FFmpegPostProcessorError as err: raise AudioConversionError(err.msg) + @PostProcessor._restrict_to(images=False) def run(self, information): path = information['filepath'] + orig_ext = information['ext'] + + if self._preferredcodec == 'best' and orig_ext in self.COMMON_AUDIO_EXTS: + self.to_screen('Skipping audio extraction since the file is already in a common audio format') + return [], information filecodec = self.get_audio_codec(path) if filecodec is None: @@ -328,11 +454,11 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly. if (new_path == path or (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))): - self._downloader.to_screen('[ffmpeg] Post-process file %s exists, skipping' % new_path) + self.to_screen('Post-process file %s exists, skipping' % new_path) return [], information try: - self._downloader.to_screen('[ffmpeg] Destination: ' + new_path) + self.to_screen('Destination: ' + new_path) self.run_ffmpeg(path, new_path, acodec, more_opts) except AudioConversionError as e: raise PostProcessingError( @@ -350,54 +476,102 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegVideoConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('mp4', 'mkv', 'flv', 'webm', 'mov', 'avi', 'mp3', 'mka', 'm4a', 'ogg', 'opus') + FORMAT_RE = re.compile(r'{0}(?:/{0})*$'.format(r'(?:\w+>)?(?:%s)' % '|'.join(SUPPORTED_EXTS))) + _ACTION = 'converting' + def __init__(self, downloader=None, preferedformat=None): super(FFmpegVideoConvertorPP, self).__init__(downloader) - self._preferedformat = preferedformat + self._preferedformats = preferedformat.lower().split('/') - def run(self, information): - path = information['filepath'] - if information['ext'] == self._preferedformat: - self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat)) - return [], information - options = [] - if self._preferedformat == 'avi': - options.extend(['-c:v', 'libxvid', '-vtag', 'XVID']) - prefix, sep, ext = path.rpartition('.') - outpath = prefix + sep + self._preferedformat - self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath) - self.run_ffmpeg(path, outpath, options) - information['filepath'] = outpath - information['format'] = self._preferedformat - information['ext'] = self._preferedformat - return [path], information + def _target_ext(self, source_ext): + for pair in self._preferedformats: + kv = pair.split('>') + if len(kv) == 1 or kv[0].strip() == source_ext: + return kv[-1].strip() + + @staticmethod + def _options(target_ext): + if target_ext == 'avi': + return ['-c:v', 'libxvid', '-vtag', 'XVID'] + return [] + + @PostProcessor._restrict_to(images=False) + def run(self, info): + filename, source_ext = info['filepath'], info['ext'].lower() + target_ext = self._target_ext(source_ext) + _skip_msg = ( + f'could not find a mapping for {source_ext}' if not target_ext + else f'already is in target format {source_ext}' if source_ext == target_ext + else None) + if _skip_msg: + self.to_screen(f'Not {self._ACTION} media file {filename!r}; {_skip_msg}') + return [], info + + outpath = replace_extension(filename, target_ext, source_ext) + self.to_screen(f'{self._ACTION.title()} video from {source_ext} to {target_ext}; Destination: {outpath}') + self.run_ffmpeg(filename, outpath, self._options(target_ext)) + + info['filepath'] = outpath + info['format'] = info['ext'] = target_ext + return [filename], info + + +class FFmpegVideoRemuxerPP(FFmpegVideoConvertorPP): + _ACTION = 'remuxing' + + @staticmethod + def _options(target_ext): + options = ['-c', 'copy', '-map', '0', '-dn'] + if target_ext in ['mp4', 'm4a', 'mov']: + options.extend(['-movflags', '+faststart']) + return options class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): + def __init__(self, downloader=None, already_have_subtitle=False): + super(FFmpegEmbedSubtitlePP, self).__init__(downloader) + self._already_have_subtitle = already_have_subtitle + + @PostProcessor._restrict_to(images=False) def run(self, information): if information['ext'] not in ('mp4', 'webm', 'mkv'): - self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files') + self.to_screen('Subtitles can only be embedded in mp4, webm or mkv files') return [], information subtitles = information.get('requested_subtitles') if not subtitles: - self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed') + self.to_screen('There aren\'t any subtitles to embed') return [], information filename = information['filepath'] + if information.get('duration') and self._duration_mismatch( + self._get_real_video_duration(information, False), information['duration']): + self.to_screen(f'Skipping {self.pp_key()} since the real and expected durations mismatch') + return [], information ext = information['ext'] - sub_langs = [] - sub_filenames = [] + sub_langs, sub_names, sub_filenames = [], [], [] webm_vtt_warn = False + mp4_ass_warn = False for lang, sub_info in subtitles.items(): + if not os.path.exists(sub_info.get('filepath', '')): + self.report_warning(f'Skipping embedding {lang} subtitle because the file is missing') + continue sub_ext = sub_info['ext'] - if ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': + if sub_ext == 'json': + self.report_warning('JSON subtitles cannot be embedded') + elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': sub_langs.append(lang) - sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext)) + sub_names.append(sub_info.get('name')) + sub_filenames.append(sub_info['filepath']) else: if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': webm_vtt_warn = True - self._downloader.to_screen('[ffmpeg] Only WebVTT subtitles can be embedded in webm files') + self.report_warning('Only WebVTT subtitles can be embedded in webm files') + if not mp4_ass_warn and ext == 'mp4' and sub_ext == 'ass': + mp4_ass_warn = True + self.report_warning('ASS subtitles cannot be properly embedded in mp4 files; expect issues') if not sub_langs: return [], information @@ -405,8 +579,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): input_files = [filename] + sub_filenames opts = [ - '-map', '0', - '-c', 'copy', + '-c', 'copy', '-map', '0', '-dn', # Don't copy the existing subtitles, we may be running the # postprocessor a second time '-map', '-0:s', @@ -416,48 +589,100 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): ] if information['ext'] == 'mp4': opts += ['-c:s', 'mov_text'] - for (i, lang) in enumerate(sub_langs): + for i, (lang, name) in enumerate(zip(sub_langs, sub_names)): opts.extend(['-map', '%d:0' % (i + 1)]) lang_code = ISO639Utils.short2long(lang) or lang opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code]) + if name: + opts.extend(['-metadata:s:s:%d' % i, 'handler_name=%s' % name, + '-metadata:s:s:%d' % i, 'title=%s' % name]) temp_filename = prepend_extension(filename, 'temp') - self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename) + self.to_screen('Embedding subtitles in "%s"' % filename) self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + os.replace(temp_filename, filename) - return sub_filenames, information + files_to_delete = [] if self._already_have_subtitle else sub_filenames + return files_to_delete, information class FFmpegMetadataPP(FFmpegPostProcessor): + + def __init__(self, downloader, add_metadata=True, add_chapters=True): + FFmpegPostProcessor.__init__(self, downloader) + self._add_metadata = add_metadata + self._add_chapters = add_chapters + + @staticmethod + def _options(target_ext): + yield from ('-map', '0', '-dn') + if target_ext == 'm4a': + yield from ('-vn', '-acodec', 'copy') + else: + yield from ('-c', 'copy') + + @PostProcessor._restrict_to(images=False) def run(self, info): + filename, metadata_filename = info['filepath'], None + options = [] + if self._add_chapters and info.get('chapters'): + metadata_filename = replace_extension(filename, 'meta') + options.extend(self._get_chapter_opts(info['chapters'], metadata_filename)) + if self._add_metadata: + options.extend(self._get_metadata_opts(info)) + + if not options: + self.to_screen('There isn\'t any metadata to add') + return [], info + + temp_filename = prepend_extension(filename, 'temp') + self.to_screen('Adding metadata to "%s"' % filename) + self.run_ffmpeg_multiple_files( + (filename, metadata_filename), temp_filename, + itertools.chain(self._options(info['ext']), *options)) + if metadata_filename: + os.remove(metadata_filename) + os.replace(temp_filename, filename) + return [], info + + @staticmethod + def _get_chapter_opts(chapters, metadata_filename): + with io.open(metadata_filename, 'wt', encoding='utf-8') as f: + def ffmpeg_escape(text): + return re.sub(r'([\\=;#\n])', r'\\\1', text) + + metadata_file_content = ';FFMETADATA1\n' + for chapter in chapters: + metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' + metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000) + metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000) + chapter_title = chapter.get('title') + if chapter_title: + metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) + f.write(metadata_file_content) + yield ('-map_metadata', '1') + + def _get_metadata_opts(self, info): metadata = {} + meta_prefix = 'meta_' def add(meta_list, info_list=None): - if not info_list: - info_list = meta_list - if not isinstance(meta_list, (list, tuple)): - meta_list = (meta_list,) - if not isinstance(info_list, (list, tuple)): - info_list = (info_list,) - for info_f in info_list: - if info.get(info_f) is not None: - for meta_f in meta_list: - metadata[meta_f] = info[info_f] - break + value = next(( + str(info[key]) for key in [meta_prefix] + list(variadic(info_list or meta_list)) + if info.get(key) is not None), None) + if value not in ('', None): + metadata.update({meta_f: value for meta_f in variadic(meta_list)}) # See [1-4] for some info on media metadata/metadata supported # by ffmpeg. # 1. https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/ # 2. https://wiki.multimedia.cx/index.php/FFmpeg_Metadata # 3. https://kodi.wiki/view/Video_file_tagging - # 4. http://atomicparsley.sourceforge.net/mpeg-4files.html add('title', ('track', 'title')) add('date', 'upload_date') - add(('description', 'comment'), 'description') - add('purl', 'webpage_url') + add(('description', 'synopsis'), 'description') + add(('purl', 'comment'), 'webpage_url') add('track', 'track_number') add('artist', ('artist', 'creator', 'uploader', 'uploader_id')) add('genre') @@ -469,57 +694,50 @@ class FFmpegMetadataPP(FFmpegPostProcessor): add('episode_id', ('episode', 'episode_id')) add('episode_sort', 'episode_number') - if not metadata: - self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add') - return [], info + for key, value in info.items(): + if value is not None and key != meta_prefix and key.startswith(meta_prefix): + metadata[key[len(meta_prefix):]] = value - filename = info['filepath'] - temp_filename = prepend_extension(filename, 'temp') - in_filenames = [filename] - options = [] + for name, value in metadata.items(): + yield ('-metadata', f'{name}={value}') - if info['ext'] == 'm4a': - options.extend(['-vn', '-acodec', 'copy']) - else: - options.extend(['-c', 'copy']) + stream_idx = 0 + for fmt in info.get('requested_formats') or []: + stream_count = 2 if 'none' not in (fmt.get('vcodec'), fmt.get('acodec')) else 1 + if fmt.get('language'): + lang = ISO639Utils.short2long(fmt['language']) or fmt['language'] + for i in range(stream_count): + yield ('-metadata:s:%d' % (stream_idx + i), 'language=%s' % lang) + stream_idx += stream_count - for (name, value) in metadata.items(): - options.extend(['-metadata', '%s=%s' % (name, value)]) + if ('no-attach-info-json' not in self.get_param('compat_opts', []) + and '__infojson_filename' in info and info['ext'] in ('mkv', 'mka')): + old_stream, new_stream = self.get_stream_number(info['filepath'], ('tags', 'mimetype'), 'application/json') + if old_stream is not None: + yield ('-map', '-0:%d' % old_stream) + new_stream -= 1 - chapters = info.get('chapters', []) - if chapters: - metadata_filename = replace_extension(filename, 'meta') - with io.open(metadata_filename, 'wt', encoding='utf-8') as f: - def ffmpeg_escape(text): - return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text) - - metadata_file_content = ';FFMETADATA1\n' - for chapter in chapters: - metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' - metadata_file_content += 'START=%d\n' % (chapter['start_time'] * 1000) - metadata_file_content += 'END=%d\n' % (chapter['end_time'] * 1000) - chapter_title = chapter.get('title') - if chapter_title: - metadata_file_content += 'title=%s\n' % ffmpeg_escape(chapter_title) - f.write(metadata_file_content) - in_filenames.append(metadata_filename) - options.extend(['-map_metadata', '1']) - - self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename) - self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options) - if chapters: - os.remove(metadata_filename) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) - return [], info + yield ('-attach', info['__infojson_filename'], + '-metadata:s:%d' % new_stream, 'mimetype=application/json') class FFmpegMergerPP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False) def run(self, info): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') - args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] - self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename) + args = ['-c', 'copy'] + audio_streams = 0 + for (i, fmt) in enumerate(info['requested_formats']): + if fmt.get('acodec') != 'none': + args.extend(['-map', f'{i}:a:0']) + aac_fixup = fmt['protocol'].startswith('m3u8') and self.get_audio_codec(fmt['filepath']) == 'aac' + if aac_fixup: + args.extend([f'-bsf:a:{audio_streams}', 'aac_adtstoasc']) + audio_streams += 1 + if fmt.get('vcodec') != 'none': + args.extend(['-map', '%u:v:0' % (i)]) + self.to_screen('Merging formats into "%s"' % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return info['__files_to_merge'], info @@ -536,98 +754,120 @@ class FFmpegMergerPP(FFmpegPostProcessor): 'hypervideo will download single file media. ' 'Update %s to version %s or newer to fix this.') % ( self.basename, self.basename, required_version) - if self._downloader: - self._downloader.report_warning(warning) + self.report_warning(warning) return False return True -class FFmpegFixupStretchedPP(FFmpegPostProcessor): - def run(self, info): - stretched_ratio = info.get('stretched_ratio') - if stretched_ratio is None or stretched_ratio == 1: - return [], info - - filename = info['filepath'] +class FFmpegFixupPostProcessor(FFmpegPostProcessor): + def _fixup(self, msg, filename, options): temp_filename = prepend_extension(filename, 'temp') - options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio] - self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename) + self.to_screen(f'{msg} of "{filename}"') self.run_ffmpeg(filename, temp_filename, options) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + os.replace(temp_filename, filename) + +class FFmpegFixupStretchedPP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False, audio=False) + def run(self, info): + stretched_ratio = info.get('stretched_ratio') + if stretched_ratio not in (None, 1): + self._fixup('Fixing aspect ratio', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-aspect', '%f' % stretched_ratio]) return [], info -class FFmpegFixupM4aPP(FFmpegPostProcessor): +class FFmpegFixupM4aPP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False, video=False) def run(self, info): - if info.get('container') != 'm4a_dash': - return [], info + if info.get('container') == 'm4a_dash': + self._fixup('Correcting container', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4']) + return [], info - filename = info['filepath'] - temp_filename = prepend_extension(filename, 'temp') - options = ['-c', 'copy', '-f', 'mp4'] - self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) +class FFmpegFixupM3u8PP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False) + def run(self, info): + if self.get_audio_codec(info['filepath']) == 'aac': + self._fixup('Fixing malformed AAC bitstream', info['filepath'], [ + '-c', 'copy', '-map', '0', '-dn', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc']) + return [], info - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) - return [], info +class FFmpegFixupTimestampPP(FFmpegFixupPostProcessor): + def __init__(self, downloader=None, trim=0.001): + # "trim" should be used when the video contains unintended packets + super(FFmpegFixupTimestampPP, self).__init__(downloader) + assert isinstance(trim, (int, float)) + self.trim = str(trim) -class FFmpegFixupM3u8PP(FFmpegPostProcessor): + @PostProcessor._restrict_to(images=False) def run(self, info): - filename = info['filepath'] - if self.get_audio_codec(filename) == 'aac': - temp_filename = prepend_extension(filename, 'temp') + required_version = '4.4' + if is_outdated_version(self._versions[self.basename], required_version): + self.report_warning( + 'A re-encode is needed to fix timestamps in older versions of ffmpeg. ' + f'Please install ffmpeg {required_version} or later to fixup without re-encoding') + opts = ['-vf', 'setpts=PTS-STARTPTS'] + else: + opts = ['-c', 'copy', '-bsf', 'setts=ts=TS-STARTPTS'] + self._fixup('Fixing frame timestamp', info['filepath'], opts + ['-map', '0', '-dn', '-ss', self.trim]) + return [], info - options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'] - self._downloader.to_screen('[ffmpeg] Fixing malformed AAC bitstream in "%s"' % filename) - self.run_ffmpeg(filename, temp_filename, options) - os.remove(encodeFilename(filename)) - os.rename(encodeFilename(temp_filename), encodeFilename(filename)) +class FFmpegFixupDurationPP(FFmpegFixupPostProcessor): + @PostProcessor._restrict_to(images=False) + def run(self, info): + self._fixup('Fixing video duration', info['filepath'], ['-c', 'copy', '-map', '0', '-dn']) return [], info class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('srt', 'vtt', 'ass', 'lrc') + def __init__(self, downloader=None, format=None): super(FFmpegSubtitlesConvertorPP, self).__init__(downloader) self.format = format def run(self, info): subs = info.get('requested_subtitles') - filename = info['filepath'] new_ext = self.format new_format = new_ext if new_format == 'vtt': new_format = 'webvtt' if subs is None: - self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert') + self.to_screen('There aren\'t any subtitles to convert') return [], info - self._downloader.to_screen('[ffmpeg] Converting subtitles') + self.to_screen('Converting subtitles') sub_filenames = [] for lang, sub in subs.items(): + if not os.path.exists(sub.get('filepath', '')): + self.report_warning(f'Skipping embedding {lang} subtitle because the file is missing') + continue ext = sub['ext'] if ext == new_ext: - self._downloader.to_screen( - '[ffmpeg] Subtitle file for %s is already in the requested format' % new_ext) + self.to_screen('Subtitle file for %s is already in the requested format' % new_ext) + continue + elif ext == 'json': + self.to_screen( + 'You have requested to convert json subtitles into another format, ' + 'which is currently not possible') continue - old_file = subtitles_filename(filename, lang, ext, info.get('ext')) + old_file = sub['filepath'] sub_filenames.append(old_file) - new_file = subtitles_filename(filename, lang, new_ext, info.get('ext')) + new_file = replace_extension(old_file, new_ext) if ext in ('dfxp', 'ttml', 'tt'): - self._downloader.report_warning( + self.report_warning( 'You have requested to convert dfxp (TTML) subtitles into another format, ' 'which results in style information loss') dfxp_file = old_file - srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext')) + srt_file = replace_extension(old_file, 'srt') with open(dfxp_file, 'rb') as f: srt_data = dfxp2srt(f.read()) @@ -638,7 +878,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': 'srt', - 'data': srt_data + 'data': srt_data, + 'filepath': srt_file, } if new_ext == 'srt': @@ -652,6 +893,125 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): subs[lang] = { 'ext': new_ext, 'data': f.read(), + 'filepath': new_file, } + info['__files_to_move'][new_file] = replace_extension( + info['__files_to_move'][sub['filepath']], new_ext) + return sub_filenames, info + + +class FFmpegSplitChaptersPP(FFmpegPostProcessor): + def __init__(self, downloader, force_keyframes=False): + FFmpegPostProcessor.__init__(self, downloader) + self._force_keyframes = force_keyframes + + def _prepare_filename(self, number, chapter, info): + info = info.copy() + info.update({ + 'section_number': number, + 'section_title': chapter.get('title'), + 'section_start': chapter.get('start_time'), + 'section_end': chapter.get('end_time'), + }) + return self._downloader.prepare_filename(info, 'chapter') + + def _ffmpeg_args_for_chapter(self, number, chapter, info): + destination = self._prepare_filename(number, chapter, info) + if not self._downloader._ensure_dir_exists(encodeFilename(destination)): + return + + chapter['filepath'] = destination + self.to_screen('Chapter %03d; Destination: %s' % (number, destination)) + return ( + destination, + ['-ss', compat_str(chapter['start_time']), + '-t', compat_str(chapter['end_time'] - chapter['start_time'])]) + + @PostProcessor._restrict_to(images=False) + def run(self, info): + chapters = info.get('chapters') or [] + if not chapters: + self.to_screen('Chapter information is unavailable') + return [], info + + in_file = info['filepath'] + if self._force_keyframes and len(chapters) > 1: + in_file = self.force_keyframes(in_file, (c['start_time'] for c in chapters)) + self.to_screen('Splitting video by chapters; %d chapters found' % len(chapters)) + for idx, chapter in enumerate(chapters): + destination, opts = self._ffmpeg_args_for_chapter(idx + 1, chapter, info) + self.real_run_ffmpeg([(in_file, opts)], [(destination, ['-c', 'copy'])]) + if in_file != info['filepath']: + os.remove(in_file) + return [], info + + +class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): + SUPPORTED_EXTS = ('jpg', 'png') + + def __init__(self, downloader=None, format=None): + super(FFmpegThumbnailsConvertorPP, self).__init__(downloader) + self.format = format + + @staticmethod + def is_webp(path): + with open(encodeFilename(path), 'rb') as f: + b = f.read(12) + return b[0:4] == b'RIFF' and b[8:] == b'WEBP' + + def fixup_webp(self, info, idx=-1): + thumbnail_filename = info['thumbnails'][idx]['filepath'] + _, thumbnail_ext = os.path.splitext(thumbnail_filename) + if thumbnail_ext: + thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext != 'webp' and self.is_webp(thumbnail_filename): + self.to_screen('Correcting thumbnail "%s" extension to webp' % thumbnail_filename) + webp_filename = replace_extension(thumbnail_filename, 'webp') + os.replace(thumbnail_filename, webp_filename) + info['thumbnails'][idx]['filepath'] = webp_filename + info['__files_to_move'][webp_filename] = replace_extension( + info['__files_to_move'].pop(thumbnail_filename), 'webp') + + @staticmethod + def _options(target_ext): + if target_ext == 'jpg': + return ['-bsf:v', 'mjpeg2jpeg'] + return [] + + def convert_thumbnail(self, thumbnail_filename, target_ext): + thumbnail_conv_filename = replace_extension(thumbnail_filename, target_ext) + + self.to_screen('Converting thumbnail "%s" to %s' % (thumbnail_filename, target_ext)) + self.real_run_ffmpeg( + [(thumbnail_filename, ['-f', 'image2', '-pattern_type', 'none'])], + [(thumbnail_conv_filename.replace('%', '%%'), self._options(target_ext))]) + return thumbnail_conv_filename + + def run(self, info): + files_to_delete = [] + has_thumbnail = False + + for idx, thumbnail_dict in enumerate(info['thumbnails']): + if 'filepath' not in thumbnail_dict: + continue + has_thumbnail = True + self.fixup_webp(info, idx) + original_thumbnail = thumbnail_dict['filepath'] + _, thumbnail_ext = os.path.splitext(original_thumbnail) + if thumbnail_ext: + thumbnail_ext = thumbnail_ext[1:].lower() + if thumbnail_ext == 'jpeg': + thumbnail_ext = 'jpg' + if thumbnail_ext == self.format: + self.to_screen('Thumbnail "%s" is already in the requested format' % original_thumbnail) + continue + thumbnail_dict['filepath'] = self.convert_thumbnail(original_thumbnail, self.format) + files_to_delete.append(original_thumbnail) + info['__files_to_move'][thumbnail_dict['filepath']] = replace_extension( + info['__files_to_move'][original_thumbnail], self.format) + + if not has_thumbnail: + self.to_screen('There aren\'t any thumbnails to convert') + return files_to_delete, info diff --git a/hypervideo_dl/postprocessor/metadataparser.py b/hypervideo_dl/postprocessor/metadataparser.py new file mode 100644 index 0000000..96aac9b --- /dev/null +++ b/hypervideo_dl/postprocessor/metadataparser.py @@ -0,0 +1,116 @@ +import re + +from enum import Enum + +from .common import PostProcessor + + +class MetadataParserPP(PostProcessor): + class Actions(Enum): + INTERPRET = 'interpretter' + REPLACE = 'replacer' + + def __init__(self, downloader, actions): + PostProcessor.__init__(self, downloader) + self._actions = [] + for f in actions: + action = f[0] + assert isinstance(action, self.Actions) + self._actions.append(getattr(self, action._value_)(*f[1:])) + + @classmethod + def validate_action(cls, action, *data): + ''' Each action can be: + (Actions.INTERPRET, from, to) OR + (Actions.REPLACE, field, search, replace) + ''' + if not isinstance(action, cls.Actions): + raise ValueError(f'{action!r} is not a valid action') + getattr(cls, action._value_)(cls, *data) + + @staticmethod + def field_to_template(tmpl): + if re.match(r'[a-zA-Z_]+$', tmpl): + return f'%({tmpl})s' + return tmpl + + @staticmethod + def format_to_regex(fmt): + r""" + Converts a string like + '%(title)s - %(artist)s' + to a regex like + '(?P<title>.+)\ \-\ (?P<artist>.+)' + """ + if not re.search(r'%\(\w+\)s', fmt): + return fmt + lastpos = 0 + regex = '' + # replace %(..)s with regex group and escape other string parts + for match in re.finditer(r'%\((\w+)\)s', fmt): + regex += re.escape(fmt[lastpos:match.start()]) + regex += rf'(?P<{match.group(1)}>.+)' + lastpos = match.end() + if lastpos < len(fmt): + regex += re.escape(fmt[lastpos:]) + return regex + + def run(self, info): + for f in self._actions: + f(info) + return [], info + + def interpretter(self, inp, out): + def f(info): + data_to_parse = self._downloader.evaluate_outtmpl(template, info) + self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}') + match = out_re.search(data_to_parse) + if match is None: + self.report_warning(f'Could not interpret {inp!r} as {out!r}') + return + for attribute, value in match.groupdict().items(): + info[attribute] = value + self.to_screen('Parsed %s from %r: %r' % (attribute, template, value if value is not None else 'NA')) + + template = self.field_to_template(inp) + out_re = re.compile(self.format_to_regex(out)) + return f + + def replacer(self, field, search, replace): + def f(info): + val = info.get(field) + if val is None: + self.report_warning(f'Video does not have a {field}') + return + elif not isinstance(val, str): + self.report_warning(f'Cannot replace in field {field} since it is a {type(val).__name__}') + return + self.write_debug(f'Replacing all {search!r} in {field} with {replace!r}') + info[field], n = search_re.subn(replace, val) + if n: + self.to_screen(f'Changed {field} to: {info[field]}') + else: + self.to_screen(f'Did not find {search!r} in {field}') + + search_re = re.compile(search) + return f + + +class MetadataFromFieldPP(MetadataParserPP): + @classmethod + def to_action(cls, f): + match = re.match(r'(?P<in>.*?)(?<!\\):(?P<out>.+)$', f) + if match is None: + raise ValueError(f'it should be FROM:TO, not {f!r}') + return ( + cls.Actions.INTERPRET, + match.group('in').replace('\\:', ':'), + match.group('out')) + + def __init__(self, downloader, formats): + MetadataParserPP.__init__(self, downloader, [self.to_action(f) for f in formats]) + + +class MetadataFromTitlePP(MetadataParserPP): # for backward compatibility + def __init__(self, downloader, titleformat): + MetadataParserPP.__init__(self, downloader, [(self.Actions.INTERPRET, 'title', titleformat)]) diff --git a/hypervideo_dl/postprocessor/modify_chapters.py b/hypervideo_dl/postprocessor/modify_chapters.py new file mode 100644 index 0000000..a0818c4 --- /dev/null +++ b/hypervideo_dl/postprocessor/modify_chapters.py @@ -0,0 +1,336 @@ +import copy +import heapq +import os + +from .common import PostProcessor +from .ffmpeg import ( + FFmpegPostProcessor, + FFmpegSubtitlesConvertorPP +) +from .sponsorblock import SponsorBlockPP +from ..utils import ( + orderedSet, + PostProcessingError, + prepend_extension, +) + + +_TINY_CHAPTER_DURATION = 1 +DEFAULT_SPONSORBLOCK_CHAPTER_TITLE = '[SponsorBlock]: %(category_names)l' + + +class ModifyChaptersPP(FFmpegPostProcessor): + def __init__(self, downloader, remove_chapters_patterns=None, remove_sponsor_segments=None, remove_ranges=None, + *, sponsorblock_chapter_title=DEFAULT_SPONSORBLOCK_CHAPTER_TITLE, force_keyframes=False): + FFmpegPostProcessor.__init__(self, downloader) + self._remove_chapters_patterns = set(remove_chapters_patterns or []) + self._remove_sponsor_segments = set(remove_sponsor_segments or []) + self._ranges_to_remove = set(remove_ranges or []) + self._sponsorblock_chapter_title = sponsorblock_chapter_title + self._force_keyframes = force_keyframes + + @PostProcessor._restrict_to(images=False) + def run(self, info): + chapters, sponsor_chapters = self._mark_chapters_to_remove( + info.get('chapters') or [], info.get('sponsorblock_chapters') or []) + if not chapters and not sponsor_chapters: + return [], info + + real_duration = self._get_real_video_duration(info) + if not chapters: + chapters = [{'start_time': 0, 'end_time': real_duration, 'title': info['title']}] + + info['chapters'], cuts = self._remove_marked_arrange_sponsors(chapters + sponsor_chapters) + if not cuts: + return [], info + + if self._duration_mismatch(real_duration, info.get('duration')): + if not self._duration_mismatch(real_duration, info['chapters'][-1]['end_time']): + self.to_screen(f'Skipping {self.pp_key()} since the video appears to be already cut') + return [], info + if not info.get('__real_download'): + raise PostProcessingError('Cannot cut video since the real and expected durations mismatch. ' + 'Different chapters may have already been removed') + else: + self.write_debug('Expected and actual durations mismatch') + + concat_opts = self._make_concat_opts(cuts, real_duration) + + def remove_chapters(file, is_sub): + return file, self.remove_chapters(file, cuts, concat_opts, self._force_keyframes and not is_sub) + + in_out_files = [remove_chapters(info['filepath'], False)] + in_out_files.extend(remove_chapters(in_file, True) for in_file in self._get_supported_subs(info)) + + # Renaming should only happen after all files are processed + files_to_remove = [] + for in_file, out_file in in_out_files: + uncut_file = prepend_extension(in_file, 'uncut') + os.replace(in_file, uncut_file) + os.replace(out_file, in_file) + files_to_remove.append(uncut_file) + + info['_real_duration'] = info['chapters'][-1]['end_time'] + return files_to_remove, info + + def _mark_chapters_to_remove(self, chapters, sponsor_chapters): + if self._remove_chapters_patterns: + warn_no_chapter_to_remove = True + if not chapters: + self.to_screen('Chapter information is unavailable') + warn_no_chapter_to_remove = False + for c in chapters: + if any(regex.search(c['title']) for regex in self._remove_chapters_patterns): + c['remove'] = True + warn_no_chapter_to_remove = False + if warn_no_chapter_to_remove: + self.to_screen('There are no chapters matching the regex') + + if self._remove_sponsor_segments: + warn_no_chapter_to_remove = True + if not sponsor_chapters: + self.to_screen('SponsorBlock information is unavailable') + warn_no_chapter_to_remove = False + for c in sponsor_chapters: + if c['category'] in self._remove_sponsor_segments: + c['remove'] = True + warn_no_chapter_to_remove = False + if warn_no_chapter_to_remove: + self.to_screen('There are no matching SponsorBlock chapters') + + sponsor_chapters.extend({ + 'start_time': start, + 'end_time': end, + 'category': 'manually_removed', + '_categories': [('manually_removed', start, end)], + 'remove': True, + } for start, end in self._ranges_to_remove) + + return chapters, sponsor_chapters + + def _get_supported_subs(self, info): + for sub in (info.get('requested_subtitles') or {}).values(): + sub_file = sub.get('filepath') + # The file might have been removed by --embed-subs + if not sub_file or not os.path.exists(sub_file): + continue + ext = sub['ext'] + if ext not in FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS: + self.report_warning(f'Cannot remove chapters from external {ext} subtitles; "{sub_file}" is now out of sync') + continue + # TODO: create __real_download for subs? + yield sub_file + + def _remove_marked_arrange_sponsors(self, chapters): + # Store cuts separately, since adjacent and overlapping cuts must be merged. + cuts = [] + + def append_cut(c): + assert 'remove' in c + last_to_cut = cuts[-1] if cuts else None + if last_to_cut and last_to_cut['end_time'] >= c['start_time']: + last_to_cut['end_time'] = max(last_to_cut['end_time'], c['end_time']) + else: + cuts.append(c) + return len(cuts) - 1 + + def excess_duration(c): + # Cuts that are completely within the chapter reduce chapters' duration. + # Since cuts can overlap, excess duration may be less that the sum of cuts' durations. + # To avoid that, chapter stores the index to the fist cut within the chapter, + # instead of storing excess duration. append_cut ensures that subsequent cuts (if any) + # will be merged with previous ones (if necessary). + cut_idx, excess = c.pop('cut_idx', len(cuts)), 0 + while cut_idx < len(cuts): + cut = cuts[cut_idx] + if cut['start_time'] >= c['end_time']: + break + if cut['end_time'] > c['start_time']: + excess += min(cut['end_time'], c['end_time']) + excess -= max(cut['start_time'], c['start_time']) + cut_idx += 1 + return excess + + new_chapters = [] + + def append_chapter(c): + assert 'remove' not in c + length = c['end_time'] - c['start_time'] - excess_duration(c) + # Chapter is completely covered by cuts or sponsors. + if length <= 0: + return + start = new_chapters[-1]['end_time'] if new_chapters else 0 + c.update(start_time=start, end_time=start + length) + new_chapters.append(c) + + # Turn into a priority queue, index is a tie breaker. + # Plain stack sorted by start_time is not enough: after splitting the chapter, + # the part returned to the stack is not guaranteed to have start_time + # less than or equal to the that of the stack's head. + chapters = [(c['start_time'], i, c) for i, c in enumerate(chapters)] + heapq.heapify(chapters) + + _, cur_i, cur_chapter = heapq.heappop(chapters) + while chapters: + _, i, c = heapq.heappop(chapters) + # Non-overlapping chapters or cuts can be appended directly. However, + # adjacent non-overlapping cuts must be merged, which is handled by append_cut. + if cur_chapter['end_time'] <= c['start_time']: + (append_chapter if 'remove' not in cur_chapter else append_cut)(cur_chapter) + cur_i, cur_chapter = i, c + continue + + # Eight possibilities for overlapping chapters: (cut, cut), (cut, sponsor), + # (cut, normal), (sponsor, cut), (normal, cut), (sponsor, sponsor), + # (sponsor, normal), and (normal, sponsor). There is no (normal, normal): + # normal chapters are assumed not to overlap. + if 'remove' in cur_chapter: + # (cut, cut): adjust end_time. + if 'remove' in c: + cur_chapter['end_time'] = max(cur_chapter['end_time'], c['end_time']) + # (cut, sponsor/normal): chop the beginning of the later chapter + # (if it's not completely hidden by the cut). Push to the priority queue + # to restore sorting by start_time: with beginning chopped, c may actually + # start later than the remaining chapters from the queue. + elif cur_chapter['end_time'] < c['end_time']: + c['start_time'] = cur_chapter['end_time'] + c['_was_cut'] = True + heapq.heappush(chapters, (c['start_time'], i, c)) + # (sponsor/normal, cut). + elif 'remove' in c: + cur_chapter['_was_cut'] = True + # Chop the end of the current chapter if the cut is not contained within it. + # Chopping the end doesn't break start_time sorting, no PQ push is necessary. + if cur_chapter['end_time'] <= c['end_time']: + cur_chapter['end_time'] = c['start_time'] + append_chapter(cur_chapter) + cur_i, cur_chapter = i, c + continue + # Current chapter contains the cut within it. If the current chapter is + # a sponsor chapter, check whether the categories before and after the cut differ. + if '_categories' in cur_chapter: + after_c = dict(cur_chapter, start_time=c['end_time'], _categories=[]) + cur_cats = [] + for cat_start_end in cur_chapter['_categories']: + if cat_start_end[1] < c['start_time']: + cur_cats.append(cat_start_end) + if cat_start_end[2] > c['end_time']: + after_c['_categories'].append(cat_start_end) + cur_chapter['_categories'] = cur_cats + if cur_chapter['_categories'] != after_c['_categories']: + # Categories before and after the cut differ: push the after part to PQ. + heapq.heappush(chapters, (after_c['start_time'], cur_i, after_c)) + cur_chapter['end_time'] = c['start_time'] + append_chapter(cur_chapter) + cur_i, cur_chapter = i, c + continue + # Either sponsor categories before and after the cut are the same or + # we're dealing with a normal chapter. Just register an outstanding cut: + # subsequent append_chapter will reduce the duration. + cur_chapter.setdefault('cut_idx', append_cut(c)) + # (sponsor, normal): if a normal chapter is not completely overlapped, + # chop the beginning of it and push it to PQ. + elif '_categories' in cur_chapter and '_categories' not in c: + if cur_chapter['end_time'] < c['end_time']: + c['start_time'] = cur_chapter['end_time'] + c['_was_cut'] = True + heapq.heappush(chapters, (c['start_time'], i, c)) + # (normal, sponsor) and (sponsor, sponsor) + else: + assert '_categories' in c + cur_chapter['_was_cut'] = True + c['_was_cut'] = True + # Push the part after the sponsor to PQ. + if cur_chapter['end_time'] > c['end_time']: + # deepcopy to make categories in after_c and cur_chapter/c refer to different lists. + after_c = dict(copy.deepcopy(cur_chapter), start_time=c['end_time']) + heapq.heappush(chapters, (after_c['start_time'], cur_i, after_c)) + # Push the part after the overlap to PQ. + elif c['end_time'] > cur_chapter['end_time']: + after_cur = dict(copy.deepcopy(c), start_time=cur_chapter['end_time']) + heapq.heappush(chapters, (after_cur['start_time'], cur_i, after_cur)) + c['end_time'] = cur_chapter['end_time'] + # (sponsor, sponsor): merge categories in the overlap. + if '_categories' in cur_chapter: + c['_categories'] = cur_chapter['_categories'] + c['_categories'] + # Inherit the cuts that the current chapter has accumulated within it. + if 'cut_idx' in cur_chapter: + c['cut_idx'] = cur_chapter['cut_idx'] + cur_chapter['end_time'] = c['start_time'] + append_chapter(cur_chapter) + cur_i, cur_chapter = i, c + (append_chapter if 'remove' not in cur_chapter else append_cut)(cur_chapter) + return self._remove_tiny_rename_sponsors(new_chapters), cuts + + def _remove_tiny_rename_sponsors(self, chapters): + new_chapters = [] + for i, c in enumerate(chapters): + # Merge with the previous/next if the chapter is tiny. + # Only tiny chapters resulting from a cut can be skipped. + # Chapters that were already tiny in the original list will be preserved. + if (('_was_cut' in c or '_categories' in c) + and c['end_time'] - c['start_time'] < _TINY_CHAPTER_DURATION): + if not new_chapters: + # Prepend tiny chapter to the next one if possible. + if i < len(chapters) - 1: + chapters[i + 1]['start_time'] = c['start_time'] + continue + else: + old_c = new_chapters[-1] + if i < len(chapters) - 1: + next_c = chapters[i + 1] + # Not a typo: key names in old_c and next_c are really different. + prev_is_sponsor = 'categories' in old_c + next_is_sponsor = '_categories' in next_c + # Preferentially prepend tiny normals to normals and sponsors to sponsors. + if (('_categories' not in c and prev_is_sponsor and not next_is_sponsor) + or ('_categories' in c and not prev_is_sponsor and next_is_sponsor)): + next_c['start_time'] = c['start_time'] + continue + old_c['end_time'] = c['end_time'] + continue + + c.pop('_was_cut', None) + cats = c.pop('_categories', None) + if cats: + category = min(cats, key=lambda c: c[2] - c[1])[0] + cats = orderedSet(x[0] for x in cats) + c.update({ + 'category': category, + 'categories': cats, + 'name': SponsorBlockPP.CATEGORIES[category], + 'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats] + }) + c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c) + # Merge identically named sponsors. + if (new_chapters and 'categories' in new_chapters[-1] + and new_chapters[-1]['title'] == c['title']): + new_chapters[-1]['end_time'] = c['end_time'] + continue + new_chapters.append(c) + return new_chapters + + def remove_chapters(self, filename, ranges_to_cut, concat_opts, force_keyframes=False): + in_file = filename + out_file = prepend_extension(in_file, 'temp') + if force_keyframes: + in_file = self.force_keyframes(in_file, (t for c in ranges_to_cut for t in (c['start_time'], c['end_time']))) + self.to_screen(f'Removing chapters from {filename}') + self.concat_files([in_file] * len(concat_opts), out_file, concat_opts) + if in_file != filename: + os.remove(in_file) + return out_file + + @staticmethod + def _make_concat_opts(chapters_to_remove, duration): + opts = [{}] + for s in chapters_to_remove: + # Do not create 0 duration chunk at the beginning. + if s['start_time'] == 0: + opts[-1]['inpoint'] = f'{s["end_time"]:.6f}' + continue + opts[-1]['outpoint'] = f'{s["start_time"]:.6f}' + # Do not create 0 duration chunk at the end. + if s['end_time'] != duration: + opts.append({'inpoint': f'{s["end_time"]:.6f}'}) + return opts diff --git a/hypervideo_dl/postprocessor/movefilesafterdownload.py b/hypervideo_dl/postprocessor/movefilesafterdownload.py new file mode 100644 index 0000000..1064a8c --- /dev/null +++ b/hypervideo_dl/postprocessor/movefilesafterdownload.py @@ -0,0 +1,54 @@ +from __future__ import unicode_literals +import os +import shutil + +from .common import PostProcessor +from ..utils import ( + decodeFilename, + encodeFilename, + make_dir, + PostProcessingError, +) + + +class MoveFilesAfterDownloadPP(PostProcessor): + + def __init__(self, downloader=None, downloaded=True): + PostProcessor.__init__(self, downloader) + self._downloaded = downloaded + + @classmethod + def pp_key(cls): + return 'MoveFiles' + + def run(self, info): + dl_path, dl_name = os.path.split(encodeFilename(info['filepath'])) + finaldir = info.get('__finaldir', dl_path) + finalpath = os.path.join(finaldir, dl_name) + if self._downloaded: + info['__files_to_move'][info['filepath']] = decodeFilename(finalpath) + + make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old)))) + for oldfile, newfile in info['__files_to_move'].items(): + if not newfile: + newfile = make_newfilename(oldfile) + if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)): + continue + if not os.path.exists(encodeFilename(oldfile)): + self.report_warning('File "%s" cannot be found' % oldfile) + continue + if os.path.exists(encodeFilename(newfile)): + if self.get_param('overwrites', True): + self.report_warning('Replacing existing file "%s"' % newfile) + os.remove(encodeFilename(newfile)) + else: + self.report_warning( + 'Cannot move file "%s" out of temporary directory since "%s" already exists. ' + % (oldfile, newfile)) + continue + make_dir(newfile, PostProcessingError) + self.to_screen('Moving file "%s" to "%s"' % (oldfile, newfile)) + shutil.move(oldfile, newfile) # os.rename cannot move between volumes + + info['filepath'] = finalpath + return [], info diff --git a/hypervideo_dl/postprocessor/sponskrub.py b/hypervideo_dl/postprocessor/sponskrub.py new file mode 100644 index 0000000..932555a --- /dev/null +++ b/hypervideo_dl/postprocessor/sponskrub.py @@ -0,0 +1,96 @@ +from __future__ import unicode_literals +import os +import subprocess + +from .common import PostProcessor +from ..compat import compat_shlex_split +from ..utils import ( + check_executable, + cli_option, + encodeArgument, + encodeFilename, + shell_quote, + str_or_none, + PostProcessingError, + prepend_extension, + process_communicate_or_kill, +) + + +# Deprecated in favor of the native implementation +class SponSkrubPP(PostProcessor): + _temp_ext = 'spons' + _exe_name = 'sponskrub' + + def __init__(self, downloader, path='', args=None, ignoreerror=False, cut=False, force=False): + PostProcessor.__init__(self, downloader) + self.force = force + self.cutout = cut + self.args = str_or_none(args) or '' # For backward compatibility + self.path = self.get_exe(path) + + if not ignoreerror and self.path is None: + if path: + raise PostProcessingError('sponskrub not found in "%s"' % path) + else: + raise PostProcessingError('sponskrub not found. Please install or provide the path using --sponskrub-path') + + def get_exe(self, path=''): + if not path or not check_executable(path, ['-h']): + path = os.path.join(path, self._exe_name) + if not check_executable(path, ['-h']): + return None + return path + + @PostProcessor._restrict_to(images=False) + def run(self, information): + if self.path is None: + return [], information + + filename = information['filepath'] + if not os.path.exists(encodeFilename(filename)): # no download + return [], information + + if information['extractor_key'].lower() != 'youtube': + self.to_screen('Skipping sponskrub since it is not a YouTube video') + return [], information + if self.cutout and not self.force and not information.get('__real_download', False): + self.report_warning( + 'Skipping sponskrub since the video was already downloaded. ' + 'Use --sponskrub-force to run sponskrub anyway') + return [], information + + self.to_screen('Trying to %s sponsor sections' % ('remove' if self.cutout else 'mark')) + if self.cutout: + self.report_warning('Cutting out sponsor segments will cause the subtitles to go out of sync.') + if not information.get('__real_download', False): + self.report_warning('If sponskrub is run multiple times, unintended parts of the video could be cut out.') + + temp_filename = prepend_extension(filename, self._temp_ext) + if os.path.exists(encodeFilename(temp_filename)): + os.remove(encodeFilename(temp_filename)) + + cmd = [self.path] + if not self.cutout: + cmd += ['-chapter'] + cmd += cli_option(self._downloader.params, '-proxy', 'proxy') + cmd += compat_shlex_split(self.args) # For backward compatibility + cmd += self._configuration_args(self._exe_name, use_compat=False) + cmd += ['--', information['id'], filename, temp_filename] + cmd = [encodeArgument(i) for i in cmd] + + self.write_debug('sponskrub command line: %s' % shell_quote(cmd)) + pipe = None if self.get_param('verbose') else subprocess.PIPE + p = subprocess.Popen(cmd, stdout=pipe) + stdout = process_communicate_or_kill(p)[0] + + if p.returncode == 0: + os.replace(temp_filename, filename) + self.to_screen('Sponsor sections have been %s' % ('removed' if self.cutout else 'marked')) + elif p.returncode == 3: + self.to_screen('No segments in the SponsorBlock database') + else: + msg = stdout.decode('utf-8', 'replace').strip() if stdout else '' + msg = msg.split('\n')[0 if msg.lower().startswith('unrecognised') else -1] + raise PostProcessingError(msg if msg else 'sponskrub failed with error code %s' % p.returncode) + return [], information diff --git a/hypervideo_dl/postprocessor/sponsorblock.py b/hypervideo_dl/postprocessor/sponsorblock.py new file mode 100644 index 0000000..7265a9d --- /dev/null +++ b/hypervideo_dl/postprocessor/sponsorblock.py @@ -0,0 +1,96 @@ +import json +import re +from hashlib import sha256 + +from .ffmpeg import FFmpegPostProcessor +from ..compat import compat_urllib_parse_urlencode, compat_HTTPError +from ..utils import PostProcessingError, network_exceptions, sanitized_Request + + +class SponsorBlockPP(FFmpegPostProcessor): + + EXTRACTORS = { + 'Youtube': 'YouTube', + } + CATEGORIES = { + 'sponsor': 'Sponsor', + 'intro': 'Intermission/Intro Animation', + 'outro': 'Endcards/Credits', + 'selfpromo': 'Unpaid/Self Promotion', + 'interaction': 'Interaction Reminder', + 'preview': 'Preview/Recap', + 'music_offtopic': 'Non-Music Section' + } + + def __init__(self, downloader, categories=None, api='https://sponsor.ajay.app'): + FFmpegPostProcessor.__init__(self, downloader) + self._categories = tuple(categories or self.CATEGORIES.keys()) + self._API_URL = api if re.match('^https?://', api) else 'https://' + api + + def run(self, info): + extractor = info['extractor_key'] + if extractor not in self.EXTRACTORS: + self.to_screen(f'SponsorBlock is not supported for {extractor}') + return [], info + + info['sponsorblock_chapters'] = self._get_sponsor_chapters(info, info['duration']) + return [], info + + def _get_sponsor_chapters(self, info, duration): + segments = self._get_sponsor_segments(info['id'], self.EXTRACTORS[info['extractor_key']]) + + def duration_filter(s): + start_end = s['segment'] + # Ignore milliseconds difference at the start. + if start_end[0] <= 1: + start_end[0] = 0 + # Ignore milliseconds difference at the end. + # Never allow the segment to exceed the video. + if duration and duration - start_end[1] <= 1: + start_end[1] = duration + # SponsorBlock duration may be absent or it may deviate from the real one. + return s['videoDuration'] == 0 or not duration or abs(duration - s['videoDuration']) <= 1 + + duration_match = [s for s in segments if duration_filter(s)] + if len(duration_match) != len(segments): + self.report_warning('Some SponsorBlock segments are from a video of different duration, maybe from an old version of this video') + + def to_chapter(s): + (start, end), cat = s['segment'], s['category'] + return { + 'start_time': start, + 'end_time': end, + 'category': cat, + 'title': self.CATEGORIES[cat], + '_categories': [(cat, start, end)] + } + + sponsor_chapters = [to_chapter(s) for s in duration_match] + if not sponsor_chapters: + self.to_screen('No segments were found in the SponsorBlock database') + else: + self.to_screen(f'Found {len(sponsor_chapters)} segments in the SponsorBlock database') + return sponsor_chapters + + def _get_sponsor_segments(self, video_id, service): + hash = sha256(video_id.encode('ascii')).hexdigest() + # SponsorBlock API recommends using first 4 hash characters. + url = f'{self._API_URL}/api/skipSegments/{hash[:4]}?' + compat_urllib_parse_urlencode({ + 'service': service, + 'categories': json.dumps(self._categories), + }) + for d in self._get_json(url): + if d['videoID'] == video_id: + return d['segments'] + return [] + + def _get_json(self, url): + self.write_debug(f'SponsorBlock query: {url}') + try: + rsp = self._downloader.urlopen(sanitized_Request(url)) + except network_exceptions as e: + if isinstance(e, compat_HTTPError) and e.code == 404: + return [] + raise PostProcessingError(f'Unable to communicate with SponsorBlock API - {e}') + + return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8')) diff --git a/hypervideo_dl/postprocessor/xattrpp.py b/hypervideo_dl/postprocessor/xattrpp.py index 814dabe..93acd6d 100644 --- a/hypervideo_dl/postprocessor/xattrpp.py +++ b/hypervideo_dl/postprocessor/xattrpp.py @@ -5,13 +5,13 @@ from ..compat import compat_os_name from ..utils import ( hyphenate_date, write_xattr, + PostProcessingError, XAttrMetadataError, XAttrUnavailableError, ) class XAttrMetadataPP(PostProcessor): - # # More info about extended attributes for media: # http://freedesktop.org/wiki/CommonExtendedAttributes/ @@ -27,7 +27,7 @@ class XAttrMetadataPP(PostProcessor): """ Set extended attributes on downloaded file (if xattr support is found). """ # Write the metadata to the file's xattrs - self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs') + self.to_screen('Writing metadata to file\'s xattrs') filename = info['filepath'] @@ -58,16 +58,15 @@ class XAttrMetadataPP(PostProcessor): return [], info except XAttrUnavailableError as e: - self._downloader.report_error(str(e)) - return [], info + raise PostProcessingError(str(e)) except XAttrMetadataError as e: if e.reason == 'NO_SPACE': - self._downloader.report_warning( + self.report_warning( 'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. ' + (('Some ' if num_written else '') + 'extended attributes are not written.').capitalize()) elif e.reason == 'VALUE_TOO_LONG': - self._downloader.report_warning( + self.report_warning( 'Unable to write extended attributes due to too long values.') else: msg = 'This filesystem doesn\'t support extended attributes. ' @@ -75,5 +74,5 @@ class XAttrMetadataPP(PostProcessor): msg += 'You need to use NTFS.' else: msg += '(You may have to enable them in your /etc/fstab)' - self._downloader.report_error(msg) + raise PostProcessingError(str(e)) return [], info |