From 4c07546e7a5e5882abdda896009b744e947df1c4 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 17 Oct 2019 19:58:13 -0700 Subject: Extraction: Replace youtube-dl with custom-built watch page extraction --- youtube_dl/options.py | 916 -------------------------------------------------- 1 file changed, 916 deletions(-) delete mode 100644 youtube_dl/options.py (limited to 'youtube_dl/options.py') diff --git a/youtube_dl/options.py b/youtube_dl/options.py deleted file mode 100644 index e7d8e89..0000000 --- a/youtube_dl/options.py +++ /dev/null @@ -1,916 +0,0 @@ -from __future__ import unicode_literals - -import os.path -import optparse -import re -import sys - -from .downloader.external import list_external_downloaders -from .compat import ( - compat_expanduser, - compat_get_terminal_size, - compat_getenv, - compat_kwargs, - compat_shlex_split, -) -from .utils import ( - preferredencoding, - write_string, -) -from .version import __version__ - - -def _hide_login_info(opts): - PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']) - eqre = re.compile('^(?P' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$') - - def _scrub_eq(o): - m = eqre.match(o) - if m: - return m.group('key') + '=PRIVATE' - else: - return o - - opts = list(map(_scrub_eq, opts)) - for idx, opt in enumerate(opts): - if opt in PRIVATE_OPTS and idx + 1 < len(opts): - opts[idx + 1] = 'PRIVATE' - return opts - - -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, default=[]): - try: - optionf = open(filename_bytes) - except IOError: - return default # silently skip if file is not present - try: - # FIXME: https://github.com/rg3/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56 - contents = optionf.read() - if sys.version_info < (3,): - contents = contents.decode(preferredencoding()) - res = compat_shlex_split(contents, comments=True) - finally: - optionf.close() - return res - - def _readUserConf(): - xdg_config_home = compat_getenv('XDG_CONFIG_HOME') - if xdg_config_home: - userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') - else: - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config') - if not os.path.isfile(userConfFile): - userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf') - userConf = _readOptions(userConfFile, None) - - if userConf is None: - appdata_dir = compat_getenv('appdata') - if appdata_dir: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), - default=None) - - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf'), - default=None) - if userConf is None: - userConf = _readOptions( - os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'), - default=None) - - if userConf is None: - userConf = [] - - return userConf - - def _format_option_string(option): - ''' ('-o', '--option') -> -o, --format METAVAR''' - - opts = [] - - if option._short_opts: - opts.append(option._short_opts[0]) - if option._long_opts: - opts.append(option._long_opts[0]) - if len(opts) > 1: - opts.insert(1, ', ') - - if option.takes_value(): - opts.append(' %s' % option.metavar) - - return ''.join(opts) - - def _comma_separated_values_options_callback(option, opt_str, value, parser): - setattr(parser.values, option.dest, value.split(',')) - - # No need to wrap help messages if we're on a wide console - columns = compat_get_terminal_size().columns - max_width = columns if columns else 80 - max_help_position = 80 - - fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) - fmt.format_option_strings = _format_option_string - - kw = { - 'version': __version__, - 'formatter': fmt, - 'usage': '%prog [OPTIONS] URL [URL...]', - 'conflict_handler': 'resolve', - } - - parser = optparse.OptionParser(**compat_kwargs(kw)) - - general = optparse.OptionGroup(parser, 'General Options') - general.add_option( - '-h', '--help', - action='help', - help='Print this help text and exit') - general.add_option( - '-v', '--version', - action='version', - help='Print program version and exit') - general.add_option( - '-U', '--update', - action='store_true', dest='update_self', - help='Update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)') - general.add_option( - '-i', '--ignore-errors', - action='store_true', dest='ignoreerrors', default=False, - help='Continue on download errors, for example to skip unavailable videos in a playlist') - general.add_option( - '--abort-on-error', - action='store_false', dest='ignoreerrors', - help='Abort downloading of further videos (in the playlist or the command line) if an error occurs') - general.add_option( - '--dump-user-agent', - action='store_true', dest='dump_user_agent', default=False, - help='Display the current browser identification') - general.add_option( - '--list-extractors', - action='store_true', dest='list_extractors', default=False, - help='List all supported extractors') - general.add_option( - '--extractor-descriptions', - action='store_true', dest='list_extractor_descriptions', default=False, - help='Output descriptions of all supported extractors') - general.add_option( - '--force-generic-extractor', - action='store_true', dest='force_generic_extractor', default=False, - help='Force extraction to use the generic extractor') - general.add_option( - '--default-search', - dest='default_search', metavar='PREFIX', - help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.') - general.add_option( - '--ignore-config', - action='store_true', - help='Do not read configuration files. ' - 'When given in the global configuration file /etc/youtube-dl.conf: ' - 'Do not read the user configuration in ~/.config/youtube-dl/config ' - '(%APPDATA%/youtube-dl/config.txt on Windows)') - general.add_option( - '--config-location', - dest='config_location', metavar='PATH', - help='Location of the configuration file; either the path to the config or its containing directory.') - general.add_option( - '--flat-playlist', - action='store_const', dest='extract_flat', const='in_playlist', - default=False, - help='Do not extract the videos of a playlist, only list them.') - general.add_option( - '--mark-watched', - action='store_true', dest='mark_watched', default=False, - help='Mark videos watched (YouTube only)') - general.add_option( - '--no-mark-watched', - action='store_false', dest='mark_watched', default=False, - help='Do not mark videos watched (YouTube only)') - general.add_option( - '--no-color', '--no-colors', - action='store_true', dest='no_color', - default=False, - help='Do not emit color codes in output') - - network = optparse.OptionGroup(parser, 'Network Options') - network.add_option( - '--proxy', dest='proxy', - default=None, metavar='URL', - help='Use the specified HTTP/HTTPS/SOCKS proxy. To enable ' - 'SOCKS proxy, specify a proper scheme. For example ' - 'socks5://127.0.0.1:1080/. Pass in an empty string (--proxy "") ' - 'for direct connection') - network.add_option( - '--socket-timeout', - dest='socket_timeout', type=float, default=None, metavar='SECONDS', - help='Time to wait before giving up, in seconds') - network.add_option( - '--source-address', - metavar='IP', dest='source_address', default=None, - help='Client-side IP address to bind to', - ) - network.add_option( - '-4', '--force-ipv4', - action='store_const', const='0.0.0.0', dest='source_address', - help='Make all connections via IPv4', - ) - network.add_option( - '-6', '--force-ipv6', - action='store_const', const='::', dest='source_address', - help='Make all connections via IPv6', - ) - - geo = optparse.OptionGroup(parser, 'Geo Restriction') - geo.add_option( - '--geo-verification-proxy', - dest='geo_verification_proxy', default=None, metavar='URL', - help='Use this proxy to verify the IP address for some geo-restricted sites. ' - 'The default proxy specified by --proxy (or none, if the option is not present) is used for the actual downloading.') - geo.add_option( - '--cn-verification-proxy', - dest='cn_verification_proxy', default=None, metavar='URL', - help=optparse.SUPPRESS_HELP) - geo.add_option( - '--geo-bypass', - action='store_true', dest='geo_bypass', default=True, - help='Bypass geographic restriction via faking X-Forwarded-For HTTP header') - geo.add_option( - '--no-geo-bypass', - action='store_false', dest='geo_bypass', default=True, - help='Do not bypass geographic restriction via faking X-Forwarded-For HTTP header') - geo.add_option( - '--geo-bypass-country', metavar='CODE', - dest='geo_bypass_country', default=None, - help='Force bypass geographic restriction with explicitly provided two-letter ISO 3166-2 country code') - geo.add_option( - '--geo-bypass-ip-block', metavar='IP_BLOCK', - dest='geo_bypass_ip_block', default=None, - help='Force bypass geographic restriction with explicitly provided IP block in CIDR notation') - - selection = optparse.OptionGroup(parser, 'Video Selection') - selection.add_option( - '--playlist-start', - dest='playliststart', metavar='NUMBER', default=1, type=int, - help='Playlist video to start at (default is %default)') - selection.add_option( - '--playlist-end', - dest='playlistend', metavar='NUMBER', default=None, type=int, - help='Playlist video to end at (default is last)') - selection.add_option( - '--playlist-items', - dest='playlist_items', metavar='ITEM_SPEC', default=None, - help='Playlist video items to download. Specify indices of the videos in the playlist separated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.') - selection.add_option( - '--match-title', - dest='matchtitle', metavar='REGEX', - help='Download only matching titles (regex or caseless sub-string)') - selection.add_option( - '--reject-title', - dest='rejecttitle', metavar='REGEX', - help='Skip download for matching titles (regex or caseless sub-string)') - selection.add_option( - '--max-downloads', - dest='max_downloads', metavar='NUMBER', type=int, default=None, - help='Abort after downloading NUMBER files') - selection.add_option( - '--min-filesize', - metavar='SIZE', dest='min_filesize', default=None, - help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)') - selection.add_option( - '--max-filesize', - metavar='SIZE', dest='max_filesize', default=None, - help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)') - selection.add_option( - '--date', - metavar='DATE', dest='date', default=None, - help='Download only videos uploaded in this date') - selection.add_option( - '--datebefore', - metavar='DATE', dest='datebefore', default=None, - help='Download only videos uploaded on or before this date (i.e. inclusive)') - selection.add_option( - '--dateafter', - metavar='DATE', dest='dateafter', default=None, - help='Download only videos uploaded on or after this date (i.e. inclusive)') - selection.add_option( - '--min-views', - metavar='COUNT', dest='min_views', default=None, type=int, - help='Do not download any videos with less than COUNT views') - selection.add_option( - '--max-views', - metavar='COUNT', dest='max_views', default=None, type=int, - help='Do not download any videos with more than COUNT views') - selection.add_option( - '--match-filter', - metavar='FILTER', dest='match_filter', default=None, - help=( - 'Generic video filter. ' - 'Specify any key (see the "OUTPUT TEMPLATE" for a list of available keys) to ' - 'match if the key is present, ' - '!key to check if the key is not present, ' - 'key > NUMBER (like "comment_count > 12", also works with ' - '>=, <, <=, !=, =) to compare against a number, ' - 'key = \'LITERAL\' (like "uploader = \'Mike Smith\'", also works with !=) ' - 'to match against a string literal ' - 'and & to require multiple matches. ' - 'Values which are not known are excluded unless you ' - 'put a question mark (?) after the operator. ' - 'For example, to only match videos that have been liked more than ' - '100 times and disliked less than 50 times (or the dislike ' - 'functionality is not available at the given service), but who ' - 'also have a description, use --match-filter ' - '"like_count > 100 & dislike_count .+?) - (?P.+)"') - postproc.add_option( - '--xattrs', - action='store_true', dest='xattrs', default=False, - help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)') - postproc.add_option( - '--fixup', - metavar='POLICY', dest='fixup', default='detect_or_warn', - help='Automatically correct known faults of the file. ' - 'One of never (do nothing), warn (only emit a warning), ' - 'detect_or_warn (the default; fix file if we can, warn otherwise)') - postproc.add_option( - '--prefer-avconv', - action='store_false', dest='prefer_ffmpeg', - help='Prefer avconv over ffmpeg for running the postprocessors') - postproc.add_option( - '--prefer-ffmpeg', - action='store_true', dest='prefer_ffmpeg', - help='Prefer ffmpeg over avconv for running the postprocessors (default)') - postproc.add_option( - '--ffmpeg-location', '--avconv-location', metavar='PATH', - dest='ffmpeg_location', - help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.') - postproc.add_option( - '--exec', - metavar='CMD', dest='exec_cmd', - help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'') - postproc.add_option( - '--convert-subs', '--convert-subtitles', - metavar='FORMAT', dest='convertsubtitles', default=None, - help='Convert the subtitles to other format (currently supported: srt|ass|vtt|lrc)') - - parser.add_option_group(general) - parser.add_option_group(network) - parser.add_option_group(geo) - parser.add_option_group(selection) - parser.add_option_group(downloader) - parser.add_option_group(filesystem) - parser.add_option_group(thumbnail) - parser.add_option_group(verbosity) - parser.add_option_group(workarounds) - parser.add_option_group(video_format) - parser.add_option_group(subtitles) - parser.add_option_group(authentication) - parser.add_option_group(adobe_pass) - parser.add_option_group(postproc) - - if overrideArguments is not None: - opts, args = parser.parse_args(overrideArguments) - if opts.verbose: - write_string('[debug] Override config: ' + repr(overrideArguments) + '\n') - else: - def compat_conf(conf): - if sys.version_info < (3,): - return [a.decode(preferredencoding(), 'replace') for a in conf] - return conf - - command_line_conf = compat_conf(sys.argv[1:]) - opts, args = parser.parse_args(command_line_conf) - - system_conf = user_conf = custom_conf = [] - - if '--config-location' in command_line_conf: - location = compat_expanduser(opts.config_location) - if os.path.isdir(location): - location = os.path.join(location, 'youtube-dl.conf') - if not os.path.exists(location): - parser.error('config-location %s does not exist.' % location) - custom_conf = _readOptions(location) - elif '--ignore-config' in command_line_conf: - pass - else: - system_conf = _readOptions('/etc/youtube-dl.conf') - if '--ignore-config' not in system_conf: - user_conf = _readUserConf() - - argv = system_conf + user_conf + custom_conf + command_line_conf - opts, args = parser.parse_args(argv) - if opts.verbose: - for conf_label, conf in ( - ('System config', system_conf), - ('User config', user_conf), - ('Custom config', custom_conf), - ('Command-line args', command_line_conf)): - write_string('[debug] %s: %s\n' % (conf_label, repr(_hide_login_info(conf)))) - - return parser, opts, args -- cgit v1.2.3