diff options
| -rw-r--r-- | README.md | 11 | ||||
| -rw-r--r-- | test/test_YoutubeDL.py | 3 | ||||
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 77 | ||||
| -rw-r--r-- | yt_dlp/__init__.py | 8 | ||||
| -rw-r--r-- | yt_dlp/compat.py | 7 | ||||
| -rw-r--r-- | yt_dlp/downloader/common.py | 86 | ||||
| -rw-r--r-- | yt_dlp/downloader/fragment.py | 4 | ||||
| -rw-r--r-- | yt_dlp/extractor/common.py | 5 | ||||
| -rw-r--r-- | yt_dlp/minicurses.py | 178 | ||||
| -rw-r--r-- | yt_dlp/options.py | 20 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/common.py | 63 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/metadataparser.py | 3 | ||||
| -rw-r--r-- | yt_dlp/postprocessor/modify_chapters.py | 3 | ||||
| -rw-r--r-- | yt_dlp/utils.py | 23 |
14 files changed, 293 insertions, 198 deletions
@@ -604,7 +604,18 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t (Alias: --force-download-archive) --newline Output progress bar as new lines --no-progress Do not print progress bar + --progress Show progress bar, even if in quiet mode --console-title Display progress in console titlebar + --progress-template [TYPES:]TEMPLATE + Template for progress outputs, optionally + prefixed with one of "download:" (default), + "download-title:" (the console title), + "postprocess:", or "postprocess-title:". + The video's fields are accessible under the + "info" key and the progress attributes are + accessible under "progress" key. Eg: + --console-title --progress-template + "download-title:%(info.id)s-%(progress.eta)s" -v, --verbose Print various debugging information --dump-pages Print downloaded pages encoded using base64 to debug problems (very verbose) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 450f25493..06963f7a8 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -666,8 +666,7 @@ class TestYoutubeDL(unittest.TestCase): ydl._num_downloads = 1 self.assertEqual(ydl.validate_outtmpl(tmpl), None) - outtmpl, tmpl_dict = ydl.prepare_outtmpl(tmpl, info or self.outtmpl_info) - out = ydl.escape_outtmpl(outtmpl) % tmpl_dict + out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info) fname = ydl.prepare_filename(info or self.outtmpl_info) if not isinstance(expected, (list, tuple)): diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 770f62734..1d865161a 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -42,6 +42,7 @@ from .compat import ( compat_urllib_error, compat_urllib_request, compat_urllib_request_DataHandler, + windows_enable_vt_mode, ) from .cookies import load_cookies from .utils import ( @@ -67,8 +68,6 @@ from .utils import ( float_or_none, format_bytes, format_field, - STR_FORMAT_RE_TMPL, - STR_FORMAT_TYPES, formatSeconds, GeoRestrictedError, HEADRequest, @@ -101,9 +100,13 @@ from .utils import ( sanitize_url, sanitized_Request, std_headers, + STR_FORMAT_RE_TMPL, + STR_FORMAT_TYPES, str_or_none, strftime_or_none, subtitles_filename, + supports_terminal_sequences, + TERMINAL_SEQUENCES, ThrottledDownload, to_high_limit_path, traverse_obj, @@ -248,6 +251,7 @@ class YoutubeDL(object): rejecttitle: Reject downloads for matching titles. logger: Log messages to a logging.Logger instance. logtostderr: Log messages to stderr instead of stdout. + consoletitle: Display progress in console window's titlebar. writedescription: Write the video description to a .description file writeinfojson: Write the video description to a .info.json file clean_infojson: Remove private fields from the infojson @@ -353,6 +357,15 @@ class YoutubeDL(object): Progress hooks are guaranteed to be called at least once (with status "finished") if the download is successful. + postprocessor_hooks: A list of functions that get called on postprocessing + progress, with a dictionary with the entries + * status: One of "started", "processing", or "finished". + Check this first and ignore unknown values. + * postprocessor: Name of the postprocessor + * info_dict: The extracted info_dict + + Progress hooks are guaranteed to be called at least twice + (with status "started" and "finished") if the processing is successful. merge_output_format: Extension to use when merging formats. final_ext: Expected final extension; used to detect when the file was already downloaded and converted. "merge_output_format" is @@ -412,11 +425,15 @@ class YoutubeDL(object): filename, abort-on-error, multistreams, no-live-chat, no-clean-infojson, no-playlist-metafiles, no-keep-subs. Refer __init__.py for their implementation + progress_template: Dictionary of templates for progress outputs. + Allowed keys are 'download', 'postprocess', + 'download-title' (console title) and 'postprocess-title'. + The template is mapped on a dictionary with keys 'progress' and 'info' The following parameters are not used by YoutubeDL itself, they are used by the downloader (see yt_dlp/downloader/common.py): nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, - max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, + max_filesize, test, noresizebuffer, retries, continuedl, noprogress, xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size. The following options are used by the post processors: @@ -484,26 +501,27 @@ class YoutubeDL(object): self._first_webpage_request = True self._post_hooks = [] self._progress_hooks = [] + self._postprocessor_hooks = [] self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._err_file = sys.stderr - self.params = { - # Default parameters - 'nocheckcertificate': False, - } - self.params.update(params) + self.params = params self.cache = Cache(self) + windows_enable_vt_mode() + self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file) + if sys.version_info < (3, 6): self.report_warning( 'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2]) if self.params.get('allow_unplayable_formats'): self.report_warning( - 'You have asked for unplayable formats to be listed/downloaded. ' - 'This is a developer option intended for debugging. ' - 'If you experience any issues while using this option, DO NOT open a bug report') + f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. ' + 'This is a developer option intended for debugging. \n' + ' If you experience any issues while using this option, ' + f'{self._color_text("DO NOT", "red")} open a bug report') def check_deprecated(param, option, suggestion): if self.params.get(param) is not None: @@ -675,9 +693,13 @@ class YoutubeDL(object): self._post_hooks.append(ph) def add_progress_hook(self, ph): - """Add the progress hook (currently only for the file downloader)""" + """Add the download progress hook""" self._progress_hooks.append(ph) + def add_postprocessor_hook(self, ph): + """Add the postprocessing progress hook""" + self._postprocessor_hooks.append(ph) + def _bidi_workaround(self, message): if not hasattr(self, '_output_channel'): return message @@ -790,6 +812,11 @@ class YoutubeDL(object): self.to_stdout( message, skip_eol, quiet=self.params.get('quiet', False)) + def _color_text(self, text, color): + if self.params.get('no_color'): + return text + return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}' + def report_warning(self, message, only_once=False): ''' Print the message to stderr, it will be prefixed with 'WARNING:' @@ -800,24 +827,14 @@ class YoutubeDL(object): else: if self.params.get('no_warnings'): return - if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': - _msg_header = '\033[0;33mWARNING:\033[0m' - else: - _msg_header = 'WARNING:' - warning_message = '%s %s' % (_msg_header, message) - self.to_stderr(warning_message, only_once) + self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once) def report_error(self, message, tb=None): ''' Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. ''' - if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt': - _msg_header = '\033[0;31mERROR:\033[0m' - else: - _msg_header = 'ERROR:' - error_message = '%s %s' % (_msg_header, message) - self.trouble(error_message, tb) + self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb) def write_debug(self, message, only_once=False): '''Log debug message or Print message to stderr''' @@ -919,7 +936,7 @@ class YoutubeDL(object): return err def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): - """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """ + """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList @@ -1073,6 +1090,10 @@ class YoutubeDL(object): return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT + def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs): + outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs) + return self.escape_outtmpl(outtmpl) % info_dict + def _prepare_filename(self, info_dict, tmpl_type='default'): try: sanitize = lambda k, v: sanitize_filename( @@ -2431,10 +2452,8 @@ class YoutubeDL(object): if self.params.get('forceprint') or self.params.get('forcejson'): self.post_extract(info_dict) for tmpl in self.params.get('forceprint', []): - if re.match(r'\w+$', tmpl): - tmpl = '%({})s'.format(tmpl) - tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict) - self.to_stdout(self.escape_outtmpl(tmpl) % info_copy) + self.to_stdout(self.evaluate_outtmpl( + f'%({tmpl})s' if re.match(r'\w+$', tmpl) else tmpl, info_dict)) print_mandatory('title') print_mandatory('id') diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 38e1d0ec6..ade822299 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -302,11 +302,14 @@ def _real_main(argv=None): parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err))) for k, tmpl in opts.outtmpl.items(): - validate_outtmpl(tmpl, '%s output template' % k) + validate_outtmpl(tmpl, f'{k} output template') opts.forceprint = opts.forceprint or [] for tmpl in opts.forceprint or []: validate_outtmpl(tmpl, 'print template') validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title') + for k, tmpl in opts.progress_template.items(): + k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress' + validate_outtmpl(tmpl, f'{k} template') if opts.extractaudio and not opts.keepvideo and opts.format is None: opts.format = 'bestaudio/best' @@ -633,8 +636,9 @@ def _real_main(argv=None): 'noresizebuffer': opts.noresizebuffer, 'http_chunk_size': opts.http_chunk_size, 'continuedl': opts.continue_dl, - 'noprogress': opts.noprogress, + 'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress, 'progress_with_newline': opts.progress_with_newline, + 'progress_template': opts.progress_template, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'playlistreverse': opts.playlist_reverse, diff --git a/yt_dlp/compat.py b/yt_dlp/compat.py index 9bf05c737..b107b2114 100644 --- a/yt_dlp/compat.py +++ b/yt_dlp/compat.py @@ -159,6 +159,12 @@ except ImportError: compat_pycrypto_AES = None +def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075 + if compat_os_name != 'nt': + return + os.system('') + + # Deprecated compat_basestring = str @@ -281,5 +287,6 @@ __all__ = [ 'compat_xml_parse_error', 'compat_xpath', 'compat_zip', + 'windows_enable_vt_mode', 'workaround_optparse_bug9161', ] diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index bb0614037..50e674829 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -7,7 +7,6 @@ import sys import time import random -from ..compat import compat_os_name from ..utils import ( decodeArgument, encodeFilename, @@ -17,6 +16,7 @@ from ..utils import ( timeconvert, ) from ..minicurses import ( + MultilineLogger, MultilinePrinter, QuietMultilinePrinter, BreaklineStatusPrinter @@ -44,8 +44,6 @@ class FileDownloader(object): noresizebuffer: Do not automatically resize the download buffer. continuedl: Try to continue downloads if possible. noprogress: Do not print the progress bar. - logtostderr: Log messages to stderr instead of stdout. - consoletitle: Display progress in console window's titlebar. nopart: Do not use temporary .part files. updatetime: Use the Last-modified header to set output file timestamps. test: Download only first bytes to test the downloader. @@ -61,6 +59,7 @@ class FileDownloader(object): http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be useful for bypassing bandwidth throttling imposed by a webserver (experimental) + progress_template: See YoutubeDL.py Subclasses of this one must re-define the real_download method. """ @@ -73,7 +72,7 @@ class FileDownloader(object): self.ydl = ydl self._progress_hooks = [] self.params = params - self._multiline = None + self._prepare_multiline_status() self.add_progress_hook(self.report_progress) @staticmethod @@ -242,55 +241,46 @@ class FileDownloader(object): """Report destination filename.""" self.to_screen('[download] Destination: ' + filename) - def _prepare_multiline_status(self, lines): - if self.params.get('quiet'): + def _prepare_multiline_status(self, lines=1): + if self.params.get('noprogress'): self._multiline = QuietMultilinePrinter() - elif self.params.get('progress_with_newline', False): + elif self.ydl.params.get('logger'): + self._multiline = MultilineLogger(self.ydl.params['logger'], lines) + elif self.params.get('progress_with_newline'): self._multiline = BreaklineStatusPrinter(sys.stderr, lines) - elif self.params.get('noprogress', False): - self._multiline = None else: - self._multiline = MultilinePrinter(sys.stderr, lines) + self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet')) def _finish_multiline_status(self): - if self._multiline is not None: - self._multiline.end() - - def _report_progress_status(self, msg, is_last_line=False, progress_line=None): - fullmsg = '[download] ' + msg - if self.params.get('progress_with_newline', False): - self.to_screen(fullmsg) - elif progress_line is not None and self._multiline is not None: - self._multiline.print_at_line(fullmsg, progress_line) - else: - if compat_os_name == 'nt' or not sys.stderr.isatty(): - prev_len = getattr(self, '_report_progress_prev_line_length', 0) - if prev_len > len(fullmsg): - fullmsg += ' ' * (prev_len - len(fullmsg)) - self._report_progress_prev_line_length = len(fullmsg) - clear_line = '\r' - else: - clear_line = '\r\x1b[K' - self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) - self.to_console_title('yt-dlp ' + msg) + self._multiline.end() + + def _report_progress_status(self, s): + progress_dict = s.copy() + progress_dict.pop('info_dict') + progress_dict = {'info': s['info_dict'], 'progress': progress_dict} + + progress_template = self.params.get('progress_template', {}) + self._multiline.print_at_line(self.ydl.evaluate_outtmpl( + progress_template.get('download') or '[download] %(progress._default_template)s', + progress_dict), s.get('progress_idx') or 0) + self.to_console_title(self.ydl.evaluate_outtmpl( + progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s', + progress_dict)) def report_progress(self, s): if s['status'] == 'finished': - if self.params.get('noprogress', False): + if self.params.get('noprogress'): self.to_screen('[download] Download completed') - else: - msg_template = '100%%' - if s.get('total_bytes') is not None: - s['_total_bytes_str'] = format_bytes(s['total_bytes']) - msg_template += ' of %(_total_bytes_str)s' - if s.get('elapsed') is not None: - s['_elapsed_str'] = self.format_seconds(s['elapsed']) - msg_template += ' in %(_elapsed_str)s' - self._report_progress_status( - msg_template % s, is_last_line=True, progress_line=s.get('progress_idx')) - return - - if self.params.get('noprogress'): + msg_template = '100%%' + if s.get('total_bytes') is not None: + s['_total_bytes_str'] = format_bytes(s['total_bytes']) + msg_template += ' of %(_total_bytes_str)s' + if s.get('elapsed') is not None: + s['_elapsed_str'] = self.format_seconds(s['elapsed']) + msg_template += ' in %(_elapsed_str)s' + s['_percent_str'] = self.format_percent(100) + s['_default_template'] = msg_template % s + self._report_progress_status(s) return if s['status'] != 'downloading': @@ -332,8 +322,8 @@ class FileDownloader(object): msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s' else: msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s' - - self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx')) + s['_default_template'] = msg_template % s + self._report_progress_status(s) def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" @@ -405,7 +395,9 @@ class FileDownloader(object): '[download] Sleeping %s seconds ...' % ( sleep_interval_sub)) time.sleep(sleep_interval_sub) - return self.real_download(filename, info_dict), True + ret = self.real_download(filename, info_dict) + self._finish_multiline_status() + return ret, True def real_download(self, filename, info_dict): """Real download process. Redefine in subclasses.""" diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 22134f3b6..6a490131b 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -393,9 +393,7 @@ class FragmentFD(FileDownloader): result = result and job.result() finally: tpe.shutdown(wait=True) - - self._finish_multiline_status() - return True + return result def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None): fragment_retries = self.params.get('fragment_retries', 0) diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index f65a098d7..4f940730a 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1134,10 +1134,7 @@ class InfoExtractor(object): if mobj: break - if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): - _name = '\033[0;34m%s\033[0m' % name - else: - _name = name + _name = self._downloader._color_text(name, 'blue') if mobj: if group is None: diff --git a/yt_dlp/minicurses.py b/yt_dlp/minicurses.py index 74ad891c9..a466fb4b0 100644 --- a/yt_dlp/minicurses.py +++ b/yt_dlp/minicurses.py @@ -1,10 +1,12 @@ -import os - from threading import Lock -from .utils import compat_os_name, get_windows_version +from .utils import supports_terminal_sequences, TERMINAL_SEQUENCES + +class MultilinePrinterBase: + def __init__(self, stream=None, lines=1): + self.stream = stream + self.maximum = lines - 1 -class MultilinePrinterBase(): def __enter__(self): return self @@ -17,119 +19,87 @@ class MultilinePrinterBase(): def end(self): pass + def _add_line_number(self, text, line): + if self.maximum: + return f'{line + 1}: {text}' + return text -class MultilinePrinter(MultilinePrinterBase): - def __init__(self, stream, lines): - """ - @param stream stream to write to - @lines number of lines to be written - """ - self.stream = stream +class QuietMultilinePrinter(MultilinePrinterBase): + pass - is_win10 = compat_os_name == 'nt' and get_windows_version() >= (10, ) - self.CARRIAGE_RETURN = '\r' - if os.getenv('TERM') and self._isatty() or is_win10: - # reason not to use curses https://github.com/yt-dlp/yt-dlp/pull/1036#discussion_r713851492 - # escape sequences for Win10 https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences - self.UP = '\x1b[A' - self.DOWN = '\n' - self.ERASE_LINE = '\x1b[K' - self._HAVE_FULLCAP = self._isatty() or is_win10 - else: - self.UP = self.DOWN = self.ERASE_LINE = None - self._HAVE_FULLCAP = False - # lines are numbered from top to bottom, counting from 0 to self.maximum - self.maximum = lines - 1 - self.lastline = 0 - self.lastlength = 0 +class MultilineLogger(MultilinePrinterBase): + def print_at_line(self, text, pos): + # stream is the logger object, not an actual stream + self.stream.debug(self._add_line_number(text, pos)) - self.movelock = Lock() - @property - def have_fullcap(self): - """ - True if the TTY is allowing to control cursor, - so that multiline progress works - """ - return self._HAVE_FULLCAP +class BreaklineStatusPrinter(MultilinePrinterBase): + def print_at_line(self, text, pos): + self.stream.write(self._add_line_number(text, pos) + '\n') - def _isatty(self): - try: - return self.stream.isatty() - except BaseException: - return False + +class MultilinePrinter(MultilinePrinterBase): + def __init__(self, stream=None, lines=1, preserve_output=True): + super().__init__(stream, lines) + self.preserve_output = preserve_output + self._lastline = self._lastlength = 0 + self._movelock = Lock() + self._HAVE_FULLCAP = supports_terminal_sequences(self.stream) + + def lock(func): + def wrapper(self, *args, **kwargs): + with self._movelock: + return func(self, *args, **kwargs) + return wrapper def _move_cursor(self, dest): - current = min(self.lastline, self.maximum) - self.stream.write(self.CARRIAGE_RETURN) - if current == dest: - # current and dest are at same position, no need to move cursor + current = min(self._lastline, self.maximum) + self.stream.write('\r') + distance = dest - current + if distance < 0: + self.stream.write(TERMINAL_SEQUENCES['UP'] * -distance) + elif distance > 0: + self.stream.write(TERMINAL_SEQUENCES['DOWN'] * distance) + self._lastline = dest + + @lock + def print_at_line(self, text, pos): + if self._HAVE_FULLCAP: + self._move_cursor(pos) + self.stream.write(TERMINAL_SEQUENCES['ERASE_LINE']) + self.stream.write(text) return - elif current > dest: - # when maximum == 2, - # 0. dest - # 1. - # 2. current - self.stream.write(self.UP * (current - dest)) - elif current < dest: - # when maximum == 2, - # 0. current - # 1. - # 2. dest - self.stream.write(self.DOWN * (dest - current)) - self.lastline = dest - def print_at_line(self, text, pos): - with self.movelock: - if self.have_fullcap: - self._move_cursor(pos) - self.stream.write(self.ERASE_LINE) - self.stream.write(text) - else: - if self.maximum != 0: - # let user know about which line is updating the status - text = f'{pos + 1}: {text}' - textlen = len(text) - if self.lastline == pos: - # move cursor at the start of progress when writing to same line - self.stream.write(self.CARRIAGE_RETURN) - if self.lastlength > textlen: - text += ' ' * (self.lastlength - textlen) - self.lastlength = textlen - else: - # otherwise, break the line - self.stream.write('\n') - self.lastlength = 0 - self.stream.write(text) - self.lastline = pos + text = self._add_line_number(text, pos) + textlen = len(text) + if self._lastline == pos: + # move cursor at the start of progress when writing to same line + self.stream.write('\r') + if self._lastlength > textlen: + text += ' ' * (self._lastlength - textlen) + self._lastlength = textlen + else: + # otherwise, break the line + self.stream.write('\n') + self._lastlength = textlen + self.stream.write(text) + self._lastline = pos + @lock def end(self): - with self.movelock: - # move cursor to the end of the last line, and write line break - # so that other to_screen calls can precede + # move cursor to the end of the last line, and write line break + # so that other to_screen calls can precede + if self._HAVE_FULLCAP: self._move_cursor(self.maximum) + if self.preserve_output: self.stream.write('\n') + return - -class QuietMultilinePrinter(MultilinePrinterBase): - def __init__(self): - self.have_fullcap = True - - -class BreaklineStatusPrinter(MultilinePrinterBase): - - def __init__(self, stream, lines): - """ - @param stream stream to write to - """ - self.stream = stream - self.maximum = lines - self.have_fullcap = True - - def print_at_line(self, text, pos): - if self.maximum != 0: - # let user know about which line is updating the status - text = f'{pos + 1}: {text}' - self.stream.write(text + '\n') + if self._HAVE_FULLCAP: + self.stream.write( + TERMINAL_SEQUENCES['ERASE_LINE'] + + f'{TERMINAL_SEQUENCES["UP"]}{TERMINAL_SEQUENCES["ERASE_LINE"]}' * self.maximum) + else: + self.stream.write(' ' * self._lastlength) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index be43f37ee..4652e8c58 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -910,13 +910,31 @@ def parseOpts(overrideArguments=None): help='Output progress bar as new lines') verbosity.add_option( '--no-progress', - action='store_true', dest='noprogress', default=False, + action='store_true', dest='noprogress', default=None, help='Do not print progress bar') verbosity.add_option( + '--progress', + action='store_false', dest='noprogress', + help='Show progress bar, even if in quiet mode') + verbosity.add_option( '--console-title', action='store_true', dest='consoletitle', default=False, help='Display progress in console titlebar') verbosity.add_option( + '--progress-template', + metavar='[TYPES:]TEMPLATE', dest='progress_template', default={}, type='str', + action='callback', callback=_dict_from_options_callback, + callback_kwargs={ + 'allowed_keys': '(download|postprocess)(-title)?', + 'default_key': 'download' + }, help=( + 'Template for progress outputs, optionally prefixed with one of "download:" (default), ' + '"download-title:" (the console title), "postprocess:", or "postprocess-title:". ' + 'The video\'s fields are accessible under the "info" key and ' + 'the progress attributes are accessible under "progress" key. Eg: ' + # TODO: Document the fields inside "progress" + '--console-title --progress-template "download-title:%(info.id)s-%(progress.eta)s"')) + verbosity.add_option( '-v', '--verbose', action='store_true', dest='verbose', default=False, help='Print various debugging information') diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py index d8ec997d9..376a1c95e 100644 --- a/yt_dlp/postprocessor/common.py +++ b/yt_dlp/postprocessor/common.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import copy import functools import os @@ -11,7 +12,26 @@ from ..utils import ( ) -class PostProcessor(object): +class PostProcessorMetaClass(type): + @staticmethod + def run_wrapper(func): + @functools.wraps(func) + def run(self, info, *args, **kwargs): + self._hook_progress({'status': 'started'}, info) + ret = func(self, info, *args, **kwargs) + if ret is not None: + _, info = ret + self._hook_progress({'status': 'finished'}, info) + return ret + return run + + def __new__(cls, name, bases, attrs): + if 'run' in attrs: + attrs['run'] = cls.run_wrapper(attrs['run']) + return type.__new__(cls, name, bases, attrs) + + +class PostProcessor(metaclass=PostProcessorMetaClass): """Post Processor class. PostProcessor objects can be added to downloaders with their @@ -34,7 +54,9 @@ class PostProcessor(object): _downloader = None def __init__(self, downloader=None): - self._downloader = downloader + self._progress_hooks = [] + self.add_progress_hook(self.report_progress) + self.set_downloader(downloader) self.PP_NAME = self.pp_key() @classmethod @@ -68,6 +90,10 @@ class PostProcessor(object): def set_downloader(self, downloader): """Sets the downloader for this PP.""" self._downloader = downloader + if not downloader: + return + for ph in downloader._postprocessor_hooks: + self.add_progress_hook(ph) @staticmethod def _restrict_to(*, video=True, audio=True, images=True): @@ -115,6 +141,39 @@ class PostProcessor(object): return _configuration_args( self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs) + def _hook_progress(self, status, info_dict): + if not self._progress_hooks: + return + info_dict = dict(info_dict) + for key in ('__original_infodict', '__postprocessors'): + info_dict.pop(key, None) + status.update({ + 'info_dict': copy.deepcopy(info_dict), + 'postprocessor': self.pp_key(), + }) + for ph in self._progress_hooks: + ph(status) + + def add_progress_hook(self, ph): + # See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface + self._progress_hooks.append(ph) + + def report_progress(self, s): + s['_default_template'] = '%(postprocessor)s %(status)s' % s + + progress_dict = s.copy() + progress_dict.pop('info_dict') + progress_dict = {'info': s['info_dict'], 'progress': progress_dict} + + progress_template = self.get_param('progress_template', {}) + tmpl = progress_template.get('postprocess') + if tmpl: + self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict)) + + self._downloader.to_console_title(self._downloader.evaluate_outtmpl( + progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s', + progress_dict)) + class AudioConversionError(PostProcessingError): pass diff --git a/yt_dlp/postprocessor/metadataparser.py b/yt_dlp/postprocessor/metadataparser.py index f7b0d8bde..96aac9beb 100644 --- a/yt_dlp/postprocessor/metadataparser.py +++ b/yt_dlp/postprocessor/metadataparser.py @@ -62,8 +62,7 @@ class MetadataParserPP(PostProcessor): def interpretter(self, inp, out): def f(info): - outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(template, info) - data_to_parse = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict + data_to_parse = self._downloader.evaluate_outtmpl(template, info) self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}') match = out_re.search(data_to_parse) if match is None: diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py index 2871e16d5..72a705fc5 100644 --- a/yt_dlp/postprocessor/modify_chapters.py +++ b/yt_dlp/postprocessor/modify_chapters.py @@ -292,8 +292,7 @@ class ModifyChaptersPP(FFmpegPostProcessor): 'name': SponsorBlockPP.CATEGORIES[category], 'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats] }) - outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c) - c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict + c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c) # Merge identically named sponsors. if (new_chapters and 'categories' in new_chapters[-1] and new_chapters[-1]['title'] == c['title']): diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 8b5b15103..027387897 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -6440,3 +6440,26 @@ def jwt_encode_hs256(payload_data, key, headers={}): signature_b64 = base64.b64encode(h.digest()) token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64 return token + + +def supports_terminal_sequences(stream): + if compat_os_name == 'nt': + if get_windows_version() < (10, ): + return False + elif not os.getenv('TERM'): + return False + try: + return stream.isatty() + except BaseException: + return False + + +TERMINAL_SEQUENCES = { + 'DOWN': '\n', + 'UP': '\x1b[A', + 'ERASE_LINE': '\x1b[K', + 'RED': '\033[0;31m', + 'YELLOW': '\033[0;33m', + 'BLUE': '\033[0;34m', + 'RESET_STYLE': '\033[0m', +} |