From 86e5f3ed2e6e71eb81ea4c9e26288f16119ffd0c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Apr 2022 20:40:28 +0530 Subject: [cleanup] Upgrade syntax Using https://github.com/asottile/pyupgrade 1. `__future__` imports and `coding: utf-8` were removed 2. Files were rewritten with `pyupgrade --py36-plus --keep-percent-format` 3. f-strings were cherry-picked from `pyupgrade --py36-plus` Extractors are left untouched (except removing header) to avoid unnecessary merge conflicts --- yt_dlp/downloader/common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index cbfea7a65..d42539931 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -1,5 +1,3 @@ -from __future__ import division, unicode_literals - import os import re import time @@ -25,7 +23,7 @@ from ..minicurses import ( ) -class FileDownloader(object): +class FileDownloader: """File Downloader class. File downloader objects are the ones responsible of downloading the @@ -219,7 +217,7 @@ class FileDownloader(object): while True: try: return func(self, *args, **kwargs) - except (IOError, OSError) as err: + except OSError as err: retry = retry + 1 if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL): if not fatal: @@ -486,4 +484,4 @@ class FileDownloader(object): if exe is None: exe = os.path.basename(str_args[0]) - self.write_debug('%s command line: %s' % (exe, shell_quote(str_args))) + self.write_debug(f'{exe} command line: {shell_quote(str_args)}') -- cgit v1.2.3 From f82711587cee043cb2496fe180b5cc0e07c06eda Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 04:02:57 +0530 Subject: [cleanup] Sort imports Using https://github.com/PyCQA/isort isort -m VERTICAL_HANGING_INDENT --py 36 -l 80 --rr -n --tc . --- yt_dlp/downloader/common.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index d42539931..3033926ae 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -1,26 +1,26 @@ +import errno import os +import random import re import time -import random -import errno +from ..minicurses import ( + BreaklineStatusPrinter, + MultilineLogger, + MultilinePrinter, + QuietMultilinePrinter, +) from ..utils import ( + LockingUnsupportedError, decodeArgument, encodeFilename, error_to_compat_str, format_bytes, - LockingUnsupportedError, sanitize_open, shell_quote, timeconvert, timetuple_from_msec, ) -from ..minicurses import ( - MultilineLogger, - MultilinePrinter, - QuietMultilinePrinter, - BreaklineStatusPrinter -) class FileDownloader: -- cgit v1.2.3 From 19a0394044bfad36cd665450271b8eb048a41c02 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Apr 2022 02:28:28 +0530 Subject: [cleanup] Misc cleanup and refactor (#2173) --- yt_dlp/downloader/common.py | 98 +++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 57 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 3033926ae..3e5396988 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -1,3 +1,4 @@ +import contextlib import errno import os import random @@ -12,6 +13,7 @@ from ..minicurses import ( ) from ..utils import ( LockingUnsupportedError, + Namespace, decodeArgument, encodeFilename, error_to_compat_str, @@ -70,12 +72,30 @@ class FileDownloader: def __init__(self, ydl, params): """Create a FileDownloader object with the given options.""" - self.ydl = ydl + self._set_ydl(ydl) self._progress_hooks = [] self.params = params self._prepare_multiline_status() self.add_progress_hook(self.report_progress) + def _set_ydl(self, ydl): + self.ydl = ydl + + for func in ( + 'deprecation_warning', + 'report_error', + 'report_file_already_downloaded', + 'report_warning', + 'to_console_title', + 'to_stderr', + 'trouble', + 'write_debug', + ): + setattr(self, func, getattr(ydl, func)) + + def to_screen(self, *args, **kargs): + self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) + @staticmethod def format_seconds(seconds): time = timetuple_from_msec(seconds * 1000) @@ -157,27 +177,6 @@ class FileDownloader: multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) return int(round(number * multiplier)) - def to_screen(self, *args, **kargs): - self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) - - def to_stderr(self, message): - self.ydl.to_stderr(message) - - def to_console_title(self, message): - self.ydl.to_console_title(message) - - def trouble(self, *args, **kargs): - self.ydl.trouble(*args, **kargs) - - def report_warning(self, *args, **kargs): - self.ydl.report_warning(*args, **kargs) - - def report_error(self, *args, **kargs): - self.ydl.report_error(*args, **kargs) - - def write_debug(self, *args, **kargs): - self.ydl.write_debug(*args, **kargs) - def slow_down(self, start_time, now, byte_counter): """Sleep if the download speed is over the rate limit.""" rate_limit = self.params.get('ratelimit') @@ -263,10 +262,8 @@ class FileDownloader: # Ignore obviously invalid dates if filetime == 0: return - try: + with contextlib.suppress(Exception): os.utime(filename, (time.time(), filetime)) - except Exception: - pass return filetime def report_destination(self, filename): @@ -287,18 +284,18 @@ class FileDownloader: def _finish_multiline_status(self): self._multiline.end() - _progress_styles = { - 'downloaded_bytes': 'light blue', - 'percent': 'light blue', - 'eta': 'yellow', - 'speed': 'green', - 'elapsed': 'bold white', - 'total_bytes': '', - 'total_bytes_estimate': '', - } + ProgressStyles = Namespace( + downloaded_bytes='light blue', + percent='light blue', + eta='yellow', + speed='green', + elapsed='bold white', + total_bytes='', + total_bytes_estimate='', + ) def _report_progress_status(self, s, default_template): - for name, style in self._progress_styles.items(): + for name, style in self.ProgressStyles._asdict().items(): name = f'_{name}_str' if name not in s: continue @@ -391,10 +388,6 @@ class FileDownloader: '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...' % (error_to_compat_str(err), count, self.format_retries(retries))) - def report_file_already_downloaded(self, *args, **kwargs): - """Report file has already been fully downloaded.""" - return self.ydl.report_file_already_downloaded(*args, **kwargs) - def report_unable_to_resume(self): """Report it was impossible to resume download.""" self.to_screen('[download] Unable to resume') @@ -433,25 +426,16 @@ class FileDownloader: self._finish_multiline_status() return True, False - if subtitle is False: - min_sleep_interval = self.params.get('sleep_interval') - if min_sleep_interval: - max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval) - sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval) - self.to_screen( - '[download] Sleeping %s seconds ...' % ( - int(sleep_interval) if sleep_interval.is_integer() - else '%.2f' % sleep_interval)) - time.sleep(sleep_interval) + if subtitle: + sleep_interval = self.params.get('sleep_interval_subtitles') or 0 else: - sleep_interval_sub = 0 - if type(self.params.get('sleep_interval_subtitles')) is int: - sleep_interval_sub = self.params.get('sleep_interval_subtitles') - if sleep_interval_sub > 0: - self.to_screen( - '[download] Sleeping %s seconds ...' % ( - sleep_interval_sub)) - time.sleep(sleep_interval_sub) + min_sleep_interval = self.params.get('sleep_interval') or 0 + sleep_interval = random.uniform( + min_sleep_interval, self.params.get('max_sleep_interval', min_sleep_interval)) + if sleep_interval > 0: + self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...') + time.sleep(sleep_interval) + ret = self.real_download(filename, info_dict) self._finish_multiline_status() return ret, True -- cgit v1.2.3 From 43cc91ad759d3950c99a905f0ee4937cade10e5c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Apr 2022 04:39:25 +0530 Subject: bugfix for 19a0394044bfad36cd665450271b8eb048a41c02, 3d3bb1688bfc5373105e6bf7c3d4729cf3f78788 --- yt_dlp/downloader/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 3e5396988..022a9cd17 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -431,7 +431,7 @@ class FileDownloader: else: min_sleep_interval = self.params.get('sleep_interval') or 0 sleep_interval = random.uniform( - min_sleep_interval, self.params.get('max_sleep_interval', min_sleep_interval)) + min_sleep_interval, self.params.get('max_sleep_interval') or min_sleep_interval) if sleep_interval > 0: self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...') time.sleep(sleep_interval) -- cgit v1.2.3 From 1d485a1a799bbeeb2faea0595676ca7d4c0f3716 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 29 Apr 2022 07:18:36 +0530 Subject: [cleanup] Misc fixes Closes #3565, https://github.com/yt-dlp/yt-dlp/issues/3514#issuecomment-1105944364 --- yt_dlp/downloader/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 022a9cd17..d79863300 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -12,6 +12,7 @@ from ..minicurses import ( QuietMultilinePrinter, ) from ..utils import ( + NUMBER_RE, LockingUnsupportedError, Namespace, decodeArgument, @@ -91,7 +92,8 @@ class FileDownloader: 'trouble', 'write_debug', ): - setattr(self, func, getattr(ydl, func)) + if not hasattr(self, func): + setattr(self, func, getattr(ydl, func)) def to_screen(self, *args, **kargs): self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) @@ -170,7 +172,7 @@ class FileDownloader: @staticmethod def parse_bytes(bytestr): """Parse a string indicating a byte quantity into an integer.""" - matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) + matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr) if matchobj is None: return None number = float(matchobj.group(1)) -- cgit v1.2.3 From 3a408f9d199127ca2626359e21a866a09ab236b3 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 11 May 2022 06:36:29 +0530 Subject: Show name of downloader in verbose log Closes #3703 --- yt_dlp/downloader/common.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'yt_dlp/downloader/common.py') diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index d79863300..1f14ebb3a 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -98,6 +98,10 @@ class FileDownloader: def to_screen(self, *args, **kargs): self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) + @property + def FD_NAME(self): + return re.sub(r'(?