diff options
| -rw-r--r-- | server.py | 36 | ||||
| -rw-r--r-- | settings.py | 44 | ||||
| -rw-r--r-- | youtube/templates/settings.html | 2 | ||||
| -rw-r--r-- | youtube/util.py | 37 | ||||
| -rw-r--r-- | youtube/watch.py | 14 |
5 files changed, 99 insertions, 34 deletions
@@ -32,24 +32,48 @@ def youtu_be(env, start_response): env['QUERY_STRING'] += '&v=' + id yield from yt_app(env, start_response) -def proxy_site(env, start_response): +def proxy_site(env, start_response, video=False): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)', 'Accept': '*/*', } + if 'HTTP_RANGE' in env: + headers['Range'] = env['HTTP_RANGE'] + url = "https://" + env['SERVER_NAME'] + env['PATH_INFO'] if env['QUERY_STRING']: url += '?' + env['QUERY_STRING'] - - content, response = util.fetch_url(url, headers, return_response=True) + if video and settings.route_tor == 1: + response, cleanup_func = util.fetch_url_response(url, headers, + use_tor=False) + else: + response, cleanup_func = util.fetch_url_response(url, headers) headers = response.getheaders() if isinstance(headers, urllib3._collections.HTTPHeaderDict): headers = headers.items() - start_response('200 OK', headers ) - yield content + start_response(str(response.status) + ' ' + response.reason, headers) + while True: + # a bit over 3 seconds of 360p video + # we want each TCP packet to transmit in large multiples, + # such as 65,536, so we shouldn't read in small chunks + # such as 8192 lest that causes the socket library to limit the + # TCP window size + # Might need fine-tuning, since this gives us 4*65536 + # The tradeoff is that larger values (such as 6 seconds) only + # allows video to buffer in those increments, meaning user must wait + # until the entire chunk is downloaded before video starts playing + content_part = response.read(32*8192) + if not content_part: + break + yield content_part + + cleanup_func(response) + +def proxy_video(env, start_response): + yield from proxy_site(env, start_response, video=True) site_handlers = { 'youtube.com':yt_app, @@ -57,7 +81,7 @@ site_handlers = { 'ytimg.com': proxy_site, 'yt3.ggpht.com': proxy_site, 'lh3.googleusercontent.com': proxy_site, - + 'googlevideo.com': proxy_video, } def split_url(url): diff --git a/settings.py b/settings.py index a6c2d55..dd39c9b 100644 --- a/settings.py +++ b/settings.py @@ -9,10 +9,17 @@ from flask import request SETTINGS_INFO = collections.OrderedDict([ ('route_tor', { - 'type': bool, - 'default': False, + 'type': int, + 'default': 0, 'label': 'Route Tor', - 'comment': '', + 'comment': '''0 - Off +1 - On, except video +2 - On, including video (see warnings)''', + 'options': [ + (0, 'Off'), + (1, 'On, except video'), + (2, 'On, including video (see warnings)'), + ], }), ('port_number', { @@ -148,7 +155,7 @@ For security reasons, enabling this is not recommended.''', ('settings_version', { 'type': int, - 'default': 2, + 'default': 3, 'comment': '''Do not change, remove, or comment out this value, or else your settings may be lost or corrupted''', 'hidden': True, }), @@ -186,8 +193,21 @@ def upgrade_to_2(settings_dict): if 'enable_related_videos' in settings_dict: new_settings['related_videos_mode'] = int(settings_dict['enable_related_videos']) del new_settings['enable_related_videos'] + new_settings['settings_version'] = 2 + return new_settings + +def upgrade_to_3(settings_dict): + new_settings = settings_dict.copy() + if 'route_tor' in settings_dict: + new_settings['route_tor'] = int(settings_dict['route_tor']) + new_settings['settings_version'] = 3 return new_settings +upgrade_functions = { + 1: upgrade_to_2, + 2: upgrade_to_3, +} + def log_ignored_line(line_number, message): print("WARNING: Ignoring settings.txt line " + str(node.lineno) + " (" + message + ")") @@ -251,14 +271,20 @@ else: current_settings_dict[target.id] = node.value.__getattribute__(attributes[type(node.value)]) - - if 'settings_version' not in current_settings_dict: - print('Upgrading settings.txt') - current_settings_dict = add_missing_settings(upgrade_to_2(current_settings_dict)) + # upgrades + latest_version = SETTINGS_INFO['settings_version']['default'] + while current_settings_dict.get('settings_version',1) < latest_version: + current_version = current_settings_dict.get('settings_version', 1) + print('Upgrading settings.txt to version', current_version+1) + upgrade_func = upgrade_functions[current_version] + # Must add missing settings here rather than below because + # save_settings needs all settings to be present + current_settings_dict = add_missing_settings( + upgrade_func(current_settings_dict)) save_settings(current_settings_dict) # some settings not in the file, add those missing settings to the file - elif not current_settings_dict.keys() >= SETTINGS_INFO.keys(): + if not current_settings_dict.keys() >= SETTINGS_INFO.keys(): print('Adding missing settings to settings.txt') current_settings_dict = add_missing_settings(current_settings_dict) save_settings(current_settings_dict) diff --git a/youtube/templates/settings.html b/youtube/templates/settings.html index 19a2461..5d1df5f 100644 --- a/youtube/templates/settings.html +++ b/youtube/templates/settings.html @@ -4,7 +4,7 @@ {% block style %} .settings-form { margin: auto; - width: 500px; + width: 600px; margin-top:10px; padding: 10px; display: block; diff --git a/youtube/util.py b/youtube/util.py index b19f91b..77c4fb1 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -119,8 +119,11 @@ def decode_content(content, encoding_header): content = gzip.decompress(content) return content -def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False, debug_name=None): +def fetch_url_response(url, headers=(), timeout=15, data=None, + cookiejar_send=None, cookiejar_receive=None, + use_tor=True): ''' + returns response, cleanup_function When cookiejar_send is set to a CookieJar object, those cookies will be sent in the request (but cookies in response will not be merged into it) When cookiejar_receive is set to a CookieJar object, @@ -147,8 +150,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja elif not isinstance(data, bytes): data = urllib.parse.urlencode(data).encode('ascii') - start_time = time.time() - if cookiejar_send is not None or cookiejar_receive is not None: # Use urllib req = urllib.request.Request(url, data=data, headers=headers) @@ -160,19 +161,30 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja opener = urllib.request.build_opener(cookie_processor) response = opener.open(req, timeout=timeout) - response_time = time.time() - - - content = response.read() + cleanup_func = (lambda r: None) else: # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them. pool = get_pool(use_tor and settings.route_tor) - response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False) - response_time = time.time() + cleanup_func = (lambda r: r.release_conn()) + + return response, cleanup_func - content = response.read() - response.release_conn() +def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, + cookiejar_send=None, cookiejar_receive=None, use_tor=True, + debug_name=None): + start_time = time.time() + + response, cleanup_func = fetch_url_response( + url, headers, timeout=timeout, + cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive, + use_tor=use_tor) + response_time = time.time() + + content = response.read() + read_finish = time.time() + + cleanup_func(response) # release_connection for urllib3 if (response.status == 429 and content.startswith(b'<!DOCTYPE') @@ -185,7 +197,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja elif response.status >= 400: raise FetchError(str(response.status), reason=response.reason, ip=None) - read_finish = time.time() if report_text: print(report_text, ' Latency:', round(response_time - start_time,3), ' Read time:', round(read_finish - response_time,3)) content = decode_content(content, response.getheader('Content-Encoding', default='identity')) @@ -198,8 +209,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja with open(os.path.join(save_dir, debug_name), 'wb') as f: f.write(content) - if return_response: - return content, response return content def head(url, use_tor=False, report_text=None, max_redirects=10): diff --git a/youtube/watch.py b/youtube/watch.py index c1f5e1e..cedf632 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -24,7 +24,7 @@ except FileNotFoundError: def get_video_sources(info): video_sources = [] - if not settings.theater_mode: + if (not settings.theater_mode) or settings.route_tor == 2: max_resolution = 360 else: max_resolution = settings.default_resolution @@ -270,10 +270,11 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): else: info['hls_formats'] = [] - # check for 403 + # check for 403. Unnecessary for tor video routing b/c ip address is same info['invidious_used'] = False info['invidious_reload_button'] = False - if settings.route_tor and info['formats'] and info['formats'][0]['url']: + if (settings.route_tor == 1 + and info['formats'] and info['formats'][0]['url']): try: response = util.head(info['formats'][0]['url'], report_text='Checked for URL access') @@ -408,10 +409,10 @@ def get_watch_page(video_id=None): "author": info['author'], } + # prefix urls, and other post-processing not handled by yt_data_extract for item in info['related_videos']: util.prefix_urls(item) util.add_extra_html_info(item) - if info['playlist']: playlist_id = info['playlist']['id'] for item in info['playlist']['items']: @@ -423,6 +424,11 @@ def get_watch_page(video_id=None): item['url'] += '&index=' + str(item['index']) info['playlist']['author_url'] = util.prefix_url( info['playlist']['author_url']) + # Don't prefix hls_formats for now because the urls inside the manifest + # would need to be prefixed as well. + for fmt in info['formats']: + fmt['url'] = util.prefix_url(fmt['url']) + if settings.gather_googlevideo_domains: with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: |