diff options
Diffstat (limited to 'yt_dlp/YoutubeDL.py')
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 148 |
1 files changed, 109 insertions, 39 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 15995bd3d..524994ab5 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -93,6 +93,7 @@ from .utils import ( PostProcessingError, preferredencoding, prepend_extension, + ReExtractInfo, register_socks_protocols, RejectedVideoReached, render_table, @@ -109,7 +110,7 @@ from .utils import ( strftime_or_none, subtitles_filename, supports_terminal_sequences, - ThrottledDownload, + timetuple_from_msec, to_high_limit_path, traverse_obj, try_get, @@ -309,6 +310,8 @@ class YoutubeDL(object): file that is in the archive. break_on_reject: Stop the download process when encountering a video that has been filtered out. + break_per_url: Whether break_on_reject and break_on_existing + should act on each input URL as opposed to for the entire queue cookiefile: File name where cookies should be read from and dumped to cookiesfrombrowser: A tuple containing the name of the browser and the profile name/path from where cookies are loaded. @@ -330,6 +333,9 @@ class YoutubeDL(object): extract_flat: Do not resolve URLs, return the immediate result. Pass in 'in_playlist' to only show this behavior for playlist items. + wait_for_video: If given, wait for scheduled streams to become available. + The value should be a tuple containing the range + (min_secs, max_secs) to wait between retries postprocessors: A list of dictionaries, each with an entry * key: The name of the postprocessor. See yt_dlp/postprocessor/__init__.py for a list. @@ -559,6 +565,8 @@ class YoutubeDL(object): for msg in self.params.get('_warnings', []): self.report_warning(msg) + for msg in self.params.get('_deprecation_warnings', []): + self.deprecation_warning(msg) if 'list-formats' in self.params.get('compat_opts', []): self.params['listformats_table'] = False @@ -841,31 +849,31 @@ class YoutubeDL(object): class Styles(Enum): HEADERS = 'yellow' - EMPHASIS = 'blue' + EMPHASIS = 'light blue' ID = 'green' DELIM = 'blue' ERROR = 'red' WARNING = 'yellow' SUPPRESS = 'light black' - def __format_text(self, out, text, f, fallback=None, *, test_encoding=False): - assert out in ('screen', 'err') + def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_encoding=False): if test_encoding: original_text = text - handle = self._screen_file if out == 'screen' else self._err_file encoding = self.params.get('encoding') or getattr(handle, 'encoding', 'ascii') text = text.encode(encoding, 'ignore').decode(encoding) if fallback is not None and text != original_text: text = fallback if isinstance(f, self.Styles): - f = f._value_ - return format_text(text, f) if self._allow_colors[out] else text if fallback is None else fallback + f = f.value + return format_text(text, f) if allow_colors else text if fallback is None else fallback def _format_screen(self, *args, **kwargs): - return self.__format_text('screen', *args, **kwargs) + return self._format_text( + self._screen_file, self._allow_colors['screen'], *args, **kwargs) def _format_err(self, *args, **kwargs): - return self.__format_text('err', *args, **kwargs) + return self._format_text( + self._err_file, self._allow_colors['err'], *args, **kwargs) def report_warning(self, message, only_once=False): ''' @@ -879,6 +887,12 @@ class YoutubeDL(object): return self.to_stderr(f'{self._format_err("WARNING:", self.Styles.WARNING)} {message}', only_once) + def deprecation_warning(self, message): + if self.params.get('logger') is not None: + self.params['logger'].warning('DeprecationWarning: {message}') + else: + self.to_stderr(f'{self._format_err("DeprecationWarning:", self.Styles.ERROR)} {message}', True) + def report_error(self, message, tb=None): ''' Do the same as trouble, but prefixes the message with 'ERROR:', colored @@ -1171,12 +1185,8 @@ class YoutubeDL(object): # https://github.com/blackjack4494/youtube-dlc/issues/85 trim_file_name = self.params.get('trim_file_name', False) if trim_file_name: - fn_groups = filename.rsplit('.') - ext = fn_groups[-1] - sub_ext = '' - if len(fn_groups) > 2: - sub_ext = fn_groups[-2] - filename = join_nonempty(fn_groups[0][:trim_file_name], sub_ext, ext, delim='.') + no_ext, *ext = filename.rsplit('.', 2) + filename = join_nonempty(no_ext[:trim_file_name], *ext, delim='.') return filename except ValueError as err: @@ -1303,8 +1313,9 @@ class YoutubeDL(object): temp_id = ie.get_temp_id(url) if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): - self.to_screen("[%s] %s: has already been recorded in archive" % ( - ie_key, temp_id)) + self.to_screen(f'[{ie_key}] {temp_id}: has already been recorded in the archive') + if self.params.get('break_on_existing', False): + raise ExistingVideoReached() break return self.__extract_info(url, self.get_info_extractor(ie_key), download, extra_info, process) else: @@ -1324,9 +1335,12 @@ class YoutubeDL(object): self.report_error(msg) except ExtractorError as e: # An error we somewhat expected self.report_error(compat_str(e), e.format_traceback()) - except ThrottledDownload as e: - self.to_stderr('\r') - self.report_warning(f'{e}; Re-extracting data') + except ReExtractInfo as e: + if e.expected: + self.to_screen(f'{e}; Re-extracting data') + else: + self.to_stderr('\r') + self.report_warning(f'{e}; Re-extracting data') return wrapper(self, *args, **kwargs) except (DownloadCancelled, LazyList.IndexError, PagedList.IndexError): raise @@ -1337,6 +1351,47 @@ class YoutubeDL(object): raise return wrapper + def _wait_for_video(self, ie_result): + if (not self.params.get('wait_for_video') + or ie_result.get('_type', 'video') != 'video' + or ie_result.get('formats') or ie_result.get('url')): + return + + format_dur = lambda dur: '%02d:%02d:%02d' % timetuple_from_msec(dur * 1000)[:-1] + last_msg = '' + + def progress(msg): + nonlocal last_msg + self.to_screen(msg + ' ' * (len(last_msg) - len(msg)) + '\r', skip_eol=True) + last_msg = msg + + min_wait, max_wait = self.params.get('wait_for_video') + diff = try_get(ie_result, lambda x: x['release_timestamp'] - time.time()) + if diff is None and ie_result.get('live_status') == 'is_upcoming': + diff = random.randrange(min_wait or 0, max_wait) if max_wait else min_wait + self.report_warning('Release time of video is not known') + elif (diff or 0) <= 0: + self.report_warning('Video should already be available according to extracted info') + diff = min(max(diff, min_wait or 0), max_wait or float('inf')) + self.to_screen(f'[wait] Waiting for {format_dur(diff)} - Press Ctrl+C to try now') + + wait_till = time.time() + diff + try: + while True: + diff = wait_till - time.time() + if diff <= 0: + progress('') + raise ReExtractInfo('[wait] Wait period ended', expected=True) + progress(f'[wait] Remaining time until next attempt: {self._format_screen(format_dur(diff), self.Styles.EMPHASIS)}') + time.sleep(1) + except KeyboardInterrupt: + progress('') + raise ReExtractInfo('[wait] Interrupted by user', expected=True) + except BaseException as e: + if not isinstance(e, ReExtractInfo): + self.to_screen('') + raise + @__handle_extraction_exceptions def __extract_info(self, url, ie, download, extra_info, process): ie_result = ie.extract(url) @@ -1352,6 +1407,7 @@ class YoutubeDL(object): ie_result.setdefault('original_url', extra_info['original_url']) self.add_default_extra_info(ie_result, ie, url) if process: + self._wait_for_video(ie_result) return self.process_ie_result(ie_result, download, extra_info) else: return ie_result @@ -2966,9 +3022,13 @@ class YoutubeDL(object): res = func(*args, **kwargs) except UnavailableVideoError as e: self.report_error(e) - except DownloadCancelled as e: + except MaxDownloadsReached as e: self.to_screen(f'[info] {e}') raise + except DownloadCancelled as e: + self.to_screen(f'[info] {e}') + if not self.params.get('break_per_url'): + raise else: if self.params.get('dump_single_json', False): self.post_extract(res) @@ -2999,7 +3059,7 @@ class YoutubeDL(object): info = self.sanitize_info(json.loads('\n'.join(f)), self.params.get('clean_infojson', True)) try: self.__download_wrapper(self.process_ie_result)(info, download=True) - except (DownloadError, EntryNotInPlaylist, ThrottledDownload) as e: + except (DownloadError, EntryNotInPlaylist, ReExtractInfo) as e: if not isinstance(e, EntryNotInPlaylist): self.to_stderr('\r') webpage_url = info.get('webpage_url') @@ -3166,15 +3226,19 @@ class YoutubeDL(object): def _format_note(self, fdict): res = '' if fdict.get('ext') in ['f4f', 'f4m']: - res += '(unsupported) ' + res += '(unsupported)' if fdict.get('language'): if res: res += ' ' - res += '[%s] ' % fdict['language'] + res += '[%s]' % fdict['language'] if fdict.get('format_note') is not None: - res += fdict['format_note'] + ' ' + if res: + res += ' ' + res += fdict['format_note'] if fdict.get('tbr') is not None: - res += '%4dk ' % fdict['tbr'] + if res: + res += ', ' + res += '%4dk' % fdict['tbr'] if fdict.get('container') is not None: if res: res += ', ' @@ -3344,7 +3408,11 @@ class YoutubeDL(object): write_debug = lambda msg: self._write_string(f'[debug] {msg}\n') source = detect_variant() - write_debug('yt-dlp version %s%s' % (__version__, '' if source == 'unknown' else f' ({source})')) + write_debug(join_nonempty( + 'yt-dlp version', __version__, + f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '', + '' if source == 'unknown' else f'({source})', + delim=' ')) if not _LAZY_LOADER: if os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'): write_debug('Lazy loading extractors is forcibly disabled') @@ -3356,20 +3424,22 @@ class YoutubeDL(object): for name, klass in itertools.chain(plugin_extractors.items(), plugin_postprocessors.items())]) if self.params.get('compat_opts'): write_debug('Compatibility options: %s' % ', '.join(self.params.get('compat_opts'))) - try: - sp = Popen( - ['git', 'rev-parse', '--short', 'HEAD'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=os.path.dirname(os.path.abspath(__file__))) - out, err = sp.communicate_or_kill() - out = out.decode().strip() - if re.match('[0-9a-f]+', out): - write_debug('Git HEAD: %s' % out) - except Exception: + + if source == 'source': try: - sys.exc_clear() + sp = Popen( + ['git', 'rev-parse', '--short', 'HEAD'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=os.path.dirname(os.path.abspath(__file__))) + out, err = sp.communicate_or_kill() + out = out.decode().strip() + if re.match('[0-9a-f]+', out): + write_debug('Git HEAD: %s' % out) except Exception: - pass + try: + sys.exc_clear() + except Exception: + pass def python_implementation(): impl_name = platform.python_implementation() |