From f4b36a220d085080a881dbe2f63e51b2fb28a003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs?= Date: Tue, 15 Dec 2020 21:14:27 -0500 Subject: pep8 --- generate_release.py | 12 +++++++++++- server.py | 17 +++++++++++------ settings.py | 24 +++++++++++++++++------- youtube/watch.py | 42 +++++++++++++++++++++++++++--------------- 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/generate_release.py b/generate_release.py index ead2772..06c348b 100644 --- a/generate_release.py +++ b/generate_release.py @@ -13,16 +13,21 @@ import hashlib latest_version = sys.argv[1] + def check(code): if code != 0: raise Exception('Got nonzero exit code from command') + + def check_subp(x): if x.returncode != 0: raise Exception('Got nonzero exit code from command') + def log(line): print('[generate_release.py] ' + line) + # https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions def remove_files_with_extensions(path, extensions): for root, dirs, files in os.walk(path): @@ -30,6 +35,7 @@ def remove_files_with_extensions(path, extensions): if os.path.splitext(file)[1] in extensions: os.remove(os.path.join(root, file)) + def download_if_not_exists(file_name, url, sha256=None): if not os.path.exists('./' + file_name): log('Downloading ' + file_name + '..') @@ -45,6 +51,7 @@ def download_if_not_exists(file_name, url, sha256=None): else: log('Using existing ' + file_name) + def wine_run_shell(command): if os.name == 'posix': check(os.system('wine ' + command.replace('\\', '/'))) @@ -53,12 +60,14 @@ def wine_run_shell(command): else: raise Exception('Unsupported OS') + def wine_run(command_parts): if os.name == 'posix': command_parts = ['wine',] + command_parts if subprocess.run(command_parts).returncode != 0: raise Exception('Got nonzero exit code from command') + # ---------- Get current release version, for later ---------- log('Getting current release version') describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE) @@ -100,7 +109,8 @@ visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a download_if_not_exists('get-pip.py', get_pip_url) download_if_not_exists('python-dist-' + latest_version + '.zip', latest_dist_url) download_if_not_exists('vc15_(14.10.25017.0)_2017_x86.7z', - visual_c_runtime_url, sha256=visual_c_runtime_sha256) + visual_c_runtime_url, + sha256=visual_c_runtime_sha256) if os.path.exists('./python'): log('Removing old python distribution') diff --git a/server.py b/server.py index 04d2d58..6c39f21 100644 --- a/server.py +++ b/server.py @@ -21,8 +21,6 @@ import re import sys - - def youtu_be(env, start_response): id = env['PATH_INFO'][1:] env['PATH_INFO'] = '/watch' @@ -32,6 +30,7 @@ def youtu_be(env, start_response): env['QUERY_STRING'] += '&v=' + id yield from yt_app(env, start_response) + def proxy_site(env, start_response, video=False): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)', @@ -79,12 +78,14 @@ def proxy_site(env, start_response, video=False): cleanup_func(response) + def proxy_video(env, start_response): yield from proxy_site(env, start_response, video=True) + site_handlers = { - 'youtube.com':yt_app, - 'youtu.be':youtu_be, + 'youtube.com': yt_app, + 'youtu.be': youtu_be, 'ytimg.com': proxy_site, 'yt3.ggpht.com': proxy_site, 'lh3.googleusercontent.com': proxy_site, @@ -92,6 +93,7 @@ site_handlers = { 'googlevideo.com': proxy_video, } + def split_url(url): ''' Split https://sub.example.com/foo/bar.html into ('sub.example.com', '/foo/bar.html')''' # XXX: Is this regex safe from REDOS? @@ -103,11 +105,11 @@ def split_url(url): return match.group(1), match.group(2) - def error_code(code, start_response): start_response(code, ()) return code.encode() + def site_dispatch(env, start_response): client_address = env['REMOTE_ADDR'] try: @@ -117,7 +119,7 @@ def site_dispatch(env, start_response): method = env['REQUEST_METHOD'] path = env['PATH_INFO'] - if method=="POST" and client_address not in ('127.0.0.1', '::1'): + if method == "POST" and client_address not in ('127.0.0.1', '::1'): yield error_code('403 Forbidden', start_response) return @@ -159,12 +161,15 @@ def site_dispatch(env, start_response): class FilteredRequestLog: '''Don't log noisy thumbnail and avatar requests''' filter_re = re.compile(r'"GET /https://(i\.ytimg\.com/|www\.youtube\.com/data/subscription_thumbnails/|yt3\.ggpht\.com/|www\.youtube\.com/api/timedtext).*" 200') + def __init__(self): pass + def write(self, s): if not self.filter_re.search(s): sys.stderr.write(s) + if __name__ == '__main__': if settings.allow_foreign_addresses: server = WSGIServer(('0.0.0.0', settings.port_number), site_dispatch, diff --git a/settings.py b/settings.py index 0b6f578..0d52bc5 100644 --- a/settings.py +++ b/settings.py @@ -235,19 +235,23 @@ def comment_string(comment): result += '# ' + line + '\n' return result + def save_settings(settings_dict): with open(settings_file_path, 'w', encoding='utf-8') as file: for setting_name, setting_info in SETTINGS_INFO.items(): file.write(comment_string(setting_info['comment']) + setting_name + ' = ' + repr(settings_dict[setting_name]) + '\n\n') + def add_missing_settings(settings_dict): result = default_settings() result.update(settings_dict) return result + def default_settings(): return {key: setting_info['default'] for key, setting_info in SETTINGS_INFO.items()} + def upgrade_to_2(settings_dict): '''Upgrade to settings version 2''' new_settings = settings_dict.copy() @@ -260,6 +264,7 @@ def upgrade_to_2(settings_dict): new_settings['settings_version'] = 2 return new_settings + def upgrade_to_3(settings_dict): new_settings = settings_dict.copy() if 'route_tor' in settings_dict: @@ -267,17 +272,17 @@ def upgrade_to_3(settings_dict): new_settings['settings_version'] = 3 return new_settings + upgrade_functions = { 1: upgrade_to_2, 2: upgrade_to_3, } + def log_ignored_line(line_number, message): print("WARNING: Ignoring settings.txt line " + str(node.lineno) + " (" + message + ")") - - if os.path.isfile("settings.txt"): print("Running in portable mode") settings_dir = os.path.normpath('./') @@ -356,16 +361,15 @@ else: globals().update(current_settings_dict) - if route_tor: print("Tor routing is ON") else: print("Tor routing is OFF - your Youtube activity is NOT anonymous") +hooks = {} -hooks = {} def add_setting_changed_hook(setting, func): '''Called right before new settings take effect''' if setting in hooks: @@ -382,11 +386,16 @@ def set_img_prefix(old_value=None, value=None): img_prefix = '/' else: img_prefix = '' + + set_img_prefix() + add_setting_changed_hook('proxy_images', set_img_prefix) categories = ['network', 'interface', 'playback', 'other'] + + def settings_page(): if request.method == 'GET': settings_by_category = {categ: [] for categ in categories} @@ -395,9 +404,10 @@ def settings_page(): settings_by_category[categ].append( (setting_name, setting_info, current_settings_dict[setting_name]) ) - return flask.render_template('settings.html', - categories = categories, - settings_by_category = settings_by_category, + return flask.render_template( + 'settings.html', + categories=categories, + settings_by_category=settings_by_category, ) elif request.method == 'POST': for key, value in request.values.items(): diff --git a/youtube/watch.py b/youtube/watch.py index 7f3b5be..d260815 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -46,6 +46,7 @@ def get_video_sources(info, tor_bypass=False): return video_sources + def make_caption_src(info, lang, auto=False, trans_lang=None): label = lang if auto: @@ -59,6 +60,7 @@ def make_caption_src(info, lang, auto=False, trans_lang=None): 'on': False, } + def lang_in(lang, sequence): '''Tests if the language is in sequence, with e.g. en and en-US considered the same''' if lang is None: @@ -66,6 +68,7 @@ def lang_in(lang, sequence): lang = lang[0:2] return lang in (l[0:2] for l in sequence) + def lang_eq(lang1, lang2): '''Tests if two iso 639-1 codes are equal, with en and en-US considered the same. Just because the codes are equal does not mean the dialects are mutually intelligible, but this will have to do for now without a complex language model''' @@ -73,6 +76,7 @@ def lang_eq(lang1, lang2): return False return lang1[0:2] == lang2[0:2] + def equiv_lang_in(lang, sequence): '''Extracts a language in sequence which is equivalent to lang. e.g. if lang is en, extracts en-GB from sequence. @@ -83,6 +87,7 @@ def equiv_lang_in(lang, sequence): return l return None + def get_subtitle_sources(info): '''Returns these sources, ordered from least to most intelligible: native_video_lang (Automatic) @@ -167,6 +172,7 @@ def get_ordered_music_list_attributes(music_list): return ordered_attributes + def save_decrypt_cache(): try: f = open(os.path.join(settings.data_dir, 'decrypt_function_cache.json'), 'w') @@ -177,6 +183,7 @@ def save_decrypt_cache(): f.write(json.dumps({'version': 1, 'decrypt_cache':decrypt_cache}, indent=4, sort_keys=True)) f.close() + watch_headers = ( ('Accept', '*/*'), ('Accept-Language', 'en-US,en;q=0.5'), @@ -184,6 +191,7 @@ watch_headers = ( ('X-YouTube-Client-Version', '2.20180830'), ) + util.mobile_ua + def decrypt_signatures(info, video_id): '''return error string, or False if no errors''' if not yt_data_extract.requires_decryption(info): @@ -206,6 +214,7 @@ def decrypt_signatures(info, video_id): err = yt_data_extract.decrypt_signatures(info) return err + def extract_info(video_id, use_invidious, playlist_id=None, index=None): # bpctr=9999999999 will bypass are-you-sure dialogs for controversial # videos @@ -255,7 +264,8 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): if (info['hls_manifest_url'] and (info['live'] or not info['formats'] or not info['urls_ready']) ): - manifest = util.fetch_url(info['hls_manifest_url'], + manifest = util.fetch_url( + info['hls_manifest_url'], debug_name='hls_manifest.m3u8', report_text='Fetched hls manifest' ).decode('utf-8') @@ -276,7 +286,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): and info['formats'] and info['formats'][0]['url']): try: response = util.head(info['formats'][0]['url'], - report_text='Checked for URL access') + report_text='Checked for URL access') except urllib3.exceptions.HTTPError: print('Error while checking for URL access:\n') traceback.print_exc() @@ -292,6 +302,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): print('Error: exceeded max redirects while checking video URL') return info + def video_quality_string(format): if format['vcodec']: result =str(format['width'] or '?') + 'x' + str(format['height'] or '?') @@ -303,6 +314,7 @@ def video_quality_string(format): return '?' + def audio_quality_string(format): if format['acodec']: result = str(format['audio_bitrate'] or '?') + 'k' @@ -314,6 +326,7 @@ def audio_quality_string(format): return '?' + # from https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py def format_bytes(bytes): if bytes is None: @@ -330,6 +343,8 @@ def format_bytes(bytes): time_table = {'h': 3600, 'm': 60, 's': 1} + + @yt_app.route('/watch') @yt_app.route('/embed') @yt_app.route('/embed/') @@ -354,15 +369,15 @@ def get_watch_page(video_id=None): use_invidious = bool(int(request.args.get('use_invidious', '1'))) tasks = ( gevent.spawn(comments.video_comments, video_id, int(settings.default_comment_sorting), lc=lc ), - gevent.spawn(extract_info, video_id, use_invidious, playlist_id=playlist_id, - index=index) + gevent.spawn(extract_info, video_id, use_invidious, + playlist_id=playlist_id, index=index) ) gevent.joinall(tasks) util.check_gevent_exceptions(tasks[1]) comments_info, info = tasks[0].value, tasks[1].value if info['error']: - return flask.render_template('error.html', error_message = info['error']) + return flask.render_template('error.html', error_message=info['error']) video_info = { "duration": util.seconds_to_timestamp(info["duration"] or 0), @@ -409,7 +424,6 @@ def get_watch_page(video_id=None): subdomain = url[0:url.find(".googlevideo.com")] f.write(subdomain + "\n") - download_formats = [] for format in (info['formats'] + info['hls_formats']): @@ -458,9 +472,9 @@ def get_watch_page(video_id=None): }) return flask.render_template('watch.html', - header_playlist_names = local_playlist.get_playlist_names(), - uploader_channel_url = ('/' + info['author_url']) if info['author_url'] else '', - time_published = info['time_published'], + header_playlist_names = local_playlist.get_playlist_names(), + uploader_channel_url = ('/' + info['author_url']) if info['author_url'] else '', + time_published = info['time_published'], view_count = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("view_count", None)), like_count = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("like_count", None)), dislike_count = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("dislike_count", None)), @@ -523,8 +537,9 @@ def get_transcript(caption_path): msg = ('Error retrieving captions: ' + str(e) + '\n\n' + 'The caption url may have expired.') print(msg) - return flask.Response(msg, - status = e.code, + return flask.Response( + msg, + status=e.code, mimetype='text/plain;charset=UTF-8') lines = captions.splitlines() @@ -571,7 +586,4 @@ def get_transcript(caption_path): result += seg['begin'] + ' ' + seg['text'] + '\r\n' return flask.Response(result.encode('utf-8'), - mimetype='text/plain;charset=UTF-8') - - - + mimetype='text/plain;charset=UTF-8') -- cgit v1.2.3