From 591bb9d3553a4d7b453777c1e28e0948741e3b50 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 17 May 2022 18:36:29 +0530 Subject: Fix color in `-q -F` and convert `ydl._out_files`/`ydl._allow_colors` to `Namespace` Closes #3761 --- yt_dlp/downloader/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 1f14ebb3a..fad4e6664 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -282,9 +282,9 @@ class FileDownloader: elif self.ydl.params.get('logger'): self._multiline = MultilineLogger(self.ydl.params['logger'], lines) elif self.params.get('progress_with_newline'): - self._multiline = BreaklineStatusPrinter(self.ydl._out_files['screen'], lines) + self._multiline = BreaklineStatusPrinter(self.ydl._out_files.screen, lines) else: - self._multiline = MultilinePrinter(self.ydl._out_files['screen'], lines, not self.params.get('quiet')) + self._multiline = MultilinePrinter(self.ydl._out_files.screen, lines, not self.params.get('quiet')) self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color') def _finish_multiline_status(self): @@ -301,7 +301,7 @@ class FileDownloader: ) def _report_progress_status(self, s, default_template): - for name, style in self.ProgressStyles._asdict().items(): + for name, style in self.ProgressStyles.items_: name = f'_{name}_str' if name not in s: continue -- cgit v1.2.3 From 7896214c42db91bbf62853b5c7359c9e83064cf1 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 17 May 2022 22:08:12 +0530 Subject: Bugfix for 591bb9d3553a4d7b453777c1e28e0948741e3b50 Closes #3769 --- yt_dlp/downloader/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index fad4e6664..465b5ef99 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -301,7 +301,7 @@ class FileDownloader: ) def _report_progress_status(self, s, default_template): - for name, style in self.ProgressStyles.items_: + for name, style in self.ProgressStyles: name = f'_{name}_str' if name not in s: continue -- cgit v1.2.3 From 23326151c45b632c3d5948bd018e80abb370e676 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 19 May 2022 20:00:31 +0530 Subject: Add option --retry-sleep (#3059) Closes #2852 --- yt_dlp/downloader/common.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 465b5ef99..0b3383071 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -19,6 +19,7 @@ from ..utils import ( encodeFilename, error_to_compat_str, format_bytes, + int_or_none, sanitize_open, shell_quote, timeconvert, @@ -64,6 +65,7 @@ class FileDownloader: useful for bypassing bandwidth throttling imposed by a webserver (experimental) progress_template: See YoutubeDL.py + retry_sleep_functions: See YoutubeDL.py Subclasses of this one must re-define the real_download method. """ @@ -98,6 +100,8 @@ class FileDownloader: def to_screen(self, *args, **kargs): self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) + __to_screen = to_screen + @property def FD_NAME(self): return re.sub(r'(? Date: Fri, 20 May 2022 20:49:30 +0530 Subject: Bugfix for 23326151c45b632c3d5948bd018e80abb370e676 --- yt_dlp/downloader/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 0b3383071..276675532 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -18,8 +18,8 @@ from ..utils import ( decodeArgument, encodeFilename, error_to_compat_str, + float_or_none, format_bytes, - int_or_none, sanitize_open, shell_quote, timeconvert, @@ -406,9 +406,9 @@ class FileDownloader: def sleep_retry(self, retry_type, count): sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type) - delay = int_or_none(sleep_func(n=count - 1)) if sleep_func else None + delay = float_or_none(sleep_func(n=count - 1)) if sleep_func else None if delay: - self.__to_screen(f'Sleeping {delay} seconds ...') + self.__to_screen(f'Sleeping {delay:.2f} seconds ...') time.sleep(delay) return sleep_func is not None -- cgit v1.2.3 From 2762dbb17e8556140f9fff0c0aa3373c521f5e09 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 20 May 2022 20:55:21 +0530 Subject: [compat] Add `functools.cached_property` --- yt_dlp/downloader/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 276675532..d74692130 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -11,6 +11,7 @@ from ..minicurses import ( MultilinePrinter, QuietMultilinePrinter, ) +from ..compat import functools from ..utils import ( NUMBER_RE, LockingUnsupportedError, @@ -102,7 +103,7 @@ class FileDownloader: __to_screen = to_screen - @property + @functools.cached_property def FD_NAME(self): return re.sub(r'(? Date: Fri, 20 May 2022 20:55:21 +0530 Subject: Bugfix for 3a408f9d199127ca2626359e21a866a09ab236b3 --- yt_dlp/downloader/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index d74692130..93eb10f76 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -11,11 +11,11 @@ from ..minicurses import ( MultilinePrinter, QuietMultilinePrinter, ) -from ..compat import functools from ..utils import ( NUMBER_RE, LockingUnsupportedError, Namespace, + classproperty, decodeArgument, encodeFilename, error_to_compat_str, @@ -103,9 +103,9 @@ class FileDownloader: __to_screen = to_screen - @functools.cached_property - def FD_NAME(self): - return re.sub(r'(? Date: Sun, 17 Apr 2022 22:48:50 +0530 Subject: [cleanup] Misc --- yt_dlp/downloader/common.py | 1 + 1 file changed, 1 insertion(+) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 93eb10f76..0aaf51633 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -43,6 +43,7 @@ class FileDownloader: verbose: Print additional info to stdout. quiet: Do not print messages to stdout. ratelimit: Download speed limit, in bytes/sec. + continuedl: Attempt to continue downloads if possible throttledratelimit: Assume the download is being throttled below this speed (bytes/sec) retries: Number of times to retry for HTTP error 5xx file_access_retries: Number of times to retry on file access error -- cgit v1.2.3 From 11233f2afdfc55d3672a7c17ad919a3f70005c19 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 22 May 2022 21:52:26 +0530 Subject: [downloader, cleanup] Refactor `report_progress` Closes #3790 --- yt_dlp/downloader/common.py | 113 +++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 65 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 0aaf51633..e2266cfb0 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -21,10 +21,12 @@ from ..utils import ( error_to_compat_str, float_or_none, format_bytes, + join_nonempty, sanitize_open, shell_quote, timeconvert, timetuple_from_msec, + try_call, ) @@ -110,6 +112,8 @@ class FileDownloader: @staticmethod def format_seconds(seconds): + if seconds is None: + return ' Unknown' time = timetuple_from_msec(seconds * 1000) if time.hours > 99: return '--:--:--' @@ -117,6 +121,8 @@ class FileDownloader: return '%02d:%02d' % time[1:-1] return '%02d:%02d:%02d' % time[:-1] + format_eta = format_seconds + @staticmethod def calc_percent(byte_counter, data_len): if data_len is None: @@ -125,11 +131,7 @@ class FileDownloader: @staticmethod def format_percent(percent): - if percent is None: - return '---.-%' - elif percent == 100: - return '100%' - return '%6s' % ('%3.1f%%' % percent) + return ' N/A%' if percent is None else f'{percent:>5.1f}%' @staticmethod def calc_eta(start, now, total, current): @@ -143,12 +145,6 @@ class FileDownloader: rate = float(current) / dif return int((float(total) - float(current)) / rate) - @staticmethod - def format_eta(eta): - if eta is None: - return '--:--' - return FileDownloader.format_seconds(eta) - @staticmethod def calc_speed(start, now, bytes): dif = now - start @@ -158,13 +154,11 @@ class FileDownloader: @staticmethod def format_speed(speed): - if speed is None: - return '%10s' % '---b/s' - return '%10s' % ('%s/s' % format_bytes(speed)) + return ' Unknown B/s' if speed is None else f'{format_bytes(speed):>10s}/s' @staticmethod def format_retries(retries): - return 'inf' if retries == float('inf') else '%.0f' % retries + return 'inf' if retries == float('inf') else int(retries) @staticmethod def best_block_size(elapsed_time, bytes): @@ -332,63 +326,52 @@ class FileDownloader: self._multiline.stream, self._multiline.allow_colors, *args, **kwargs) def report_progress(self, s): + def with_fields(*tups, default=''): + for *fields, tmpl in tups: + if all(s.get(f) is not None for f in fields): + return tmpl + return default + if s['status'] == 'finished': if self.params.get('noprogress'): self.to_screen('[download] Download completed') - msg_template = '100%%' - if s.get('total_bytes') is not None: - s['_total_bytes_str'] = format_bytes(s['total_bytes']) - msg_template += ' of %(_total_bytes_str)s' - if s.get('elapsed') is not None: - s['_elapsed_str'] = self.format_seconds(s['elapsed']) - msg_template += ' in %(_elapsed_str)s' - s['_percent_str'] = self.format_percent(100) - self._report_progress_status(s, msg_template) - return + s.update({ + '_total_bytes_str': format_bytes(s.get('total_bytes')), + '_elapsed_str': self.format_seconds(s.get('elapsed')), + '_percent_str': self.format_percent(100), + }) + self._report_progress_status(s, join_nonempty( + '100%%', + with_fields(('total_bytes', 'of %(_total_bytes_str)s')), + with_fields(('elapsed', 'in %(_elapsed_str)s')), + delim=' ')) if s['status'] != 'downloading': return - if s.get('eta') is not None: - s['_eta_str'] = self.format_eta(s['eta']) - else: - s['_eta_str'] = 'Unknown' - - if s.get('total_bytes') and s.get('downloaded_bytes') is not None: - s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes']) - elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None: - s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate']) - else: - if s.get('downloaded_bytes') == 0: - s['_percent_str'] = self.format_percent(0) - else: - s['_percent_str'] = 'Unknown %' - - if s.get('speed') is not None: - s['_speed_str'] = self.format_speed(s['speed']) - else: - s['_speed_str'] = 'Unknown speed' - - if s.get('total_bytes') is not None: - s['_total_bytes_str'] = format_bytes(s['total_bytes']) - msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s' - elif s.get('total_bytes_estimate') is not None: - s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate']) - msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s' - else: - if s.get('downloaded_bytes') is not None: - s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes']) - if s.get('elapsed'): - s['_elapsed_str'] = self.format_seconds(s['elapsed']) - msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)' - else: - msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s' - else: - msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s' - if s.get('fragment_index') and s.get('fragment_count'): - msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)' - elif s.get('fragment_index'): - msg_template += ' (frag %(fragment_index)s)' + s.update({ + '_eta_str': self.format_eta(s.get('eta')), + '_speed_str': self.format_speed(s.get('speed')), + '_percent_str': self.format_percent(try_call( + lambda: 100 * s['downloaded_bytes'] / s['total_bytes'], + lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'], + lambda: s['downloaded_bytes'] == 0 and 0)), + '_total_bytes_str': format_bytes(s.get('total_bytes')), + '_total_bytes_estimate_str': format_bytes(s.get('total_bytes_estimate')), + '_downloaded_bytes_str': format_bytes(s.get('downloaded_bytes')), + '_elapsed_str': self.format_seconds(s.get('elapsed')), + }) + + msg_template = with_fields( + ('total_bytes', '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'), + ('total_bytes_estimate', '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'), + ('downloaded_bytes', 'elapsed', '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'), + ('downloaded_bytes', '%(_downloaded_bytes_str)s at %(_speed_str)s'), + default='%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s') + + msg_template += with_fields( + ('fragment_index', 'fragment_count', ' (frag %(fragment_index)s/%(fragment_count)s)'), + ('fragment_index', ' (frag %(fragment_index)s)')) self._report_progress_status(s, msg_template) def report_resuming_byte(self, resume_len): -- cgit v1.2.3 From 8a7f6d7a155bc0966c40736336faea81db92315b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 31 May 2022 02:23:54 +0530 Subject: Do not print progress to `stderr` with `-q` It is arguable how this "should" behave, but since progress is always written to stdout in older yt-dl/p, we should keep it as-is Bug in cf4f42cb9776eaa3166d2d234c3ec7651f05d7a9 Closes #3844 --- yt_dlp/downloader/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index e2266cfb0..cd30d1eff 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -283,9 +283,9 @@ class FileDownloader: elif self.ydl.params.get('logger'): self._multiline = MultilineLogger(self.ydl.params['logger'], lines) elif self.params.get('progress_with_newline'): - self._multiline = BreaklineStatusPrinter(self.ydl._out_files.screen, lines) + self._multiline = BreaklineStatusPrinter(self.ydl._out_files.out, lines) else: - self._multiline = MultilinePrinter(self.ydl._out_files.screen, lines, not self.params.get('quiet')) + self._multiline = MultilinePrinter(self.ydl._out_files.out, lines, not self.params.get('quiet')) self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color') def _finish_multiline_status(self): -- cgit v1.2.3 From 64fa820ccf61a7aea6c2a48b1362b3a4ec270cad Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 25 May 2022 17:53:46 +0530 Subject: [cleanup] Misc fixes (see desc) * [tvver] Fix bug in 6837633a4a614920b6e43ffc6b4b8590dca8c9d7 - Closes #4054 * [rumble] Fix tests - Closes #3976 * [make] Remove `cat` abuse - Closes #3989 * [make] Revert #3684 - Closes #3814 * [utils] Improve `get_elements_by_class` - Closes #3993 * [utils] Inherit `Namespace` from `types.SimpleNamespace` * [utils] Use `re.fullmatch` for matching filters * [jsinterp] Handle quotes in `_separate` * [make_readme] Allow overshooting last line Authored by: pukkandan, kwconder, MrRawes, Lesmiscore --- yt_dlp/downloader/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index cd30d1eff..b559e7cae 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -302,7 +302,7 @@ class FileDownloader: ) def _report_progress_status(self, s, default_template): - for name, style in self.ProgressStyles: + for name, style in self.ProgressStyles.items_: name = f'_{name}_str' if name not in s: continue -- cgit v1.2.3 From 998a3cae0ca2e2276ea02a3470c7e083feec96f5 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 22 Jun 2022 03:47:41 +0530 Subject: [cleanup] Misc fixes --- yt_dlp/downloader/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index b559e7cae..3a0a014ef 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -108,7 +108,7 @@ class FileDownloader: @classproperty def FD_NAME(cls): - return re.sub(r'(?