diff options
Diffstat (limited to 'youtube')
| -rw-r--r-- | youtube/templates/settings.html | 2 | ||||
| -rw-r--r-- | youtube/util.py | 37 | ||||
| -rw-r--r-- | youtube/watch.py | 14 |
3 files changed, 34 insertions, 19 deletions
diff --git a/youtube/templates/settings.html b/youtube/templates/settings.html index 19a2461..5d1df5f 100644 --- a/youtube/templates/settings.html +++ b/youtube/templates/settings.html @@ -4,7 +4,7 @@ {% block style %} .settings-form { margin: auto; - width: 500px; + width: 600px; margin-top:10px; padding: 10px; display: block; diff --git a/youtube/util.py b/youtube/util.py index b19f91b..77c4fb1 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -119,8 +119,11 @@ def decode_content(content, encoding_header): content = gzip.decompress(content) return content -def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False, debug_name=None): +def fetch_url_response(url, headers=(), timeout=15, data=None, + cookiejar_send=None, cookiejar_receive=None, + use_tor=True): ''' + returns response, cleanup_function When cookiejar_send is set to a CookieJar object, those cookies will be sent in the request (but cookies in response will not be merged into it) When cookiejar_receive is set to a CookieJar object, @@ -147,8 +150,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja elif not isinstance(data, bytes): data = urllib.parse.urlencode(data).encode('ascii') - start_time = time.time() - if cookiejar_send is not None or cookiejar_receive is not None: # Use urllib req = urllib.request.Request(url, data=data, headers=headers) @@ -160,19 +161,30 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja opener = urllib.request.build_opener(cookie_processor) response = opener.open(req, timeout=timeout) - response_time = time.time() - - - content = response.read() + cleanup_func = (lambda r: None) else: # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them. pool = get_pool(use_tor and settings.route_tor) - response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False) - response_time = time.time() + cleanup_func = (lambda r: r.release_conn()) + + return response, cleanup_func - content = response.read() - response.release_conn() +def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, + cookiejar_send=None, cookiejar_receive=None, use_tor=True, + debug_name=None): + start_time = time.time() + + response, cleanup_func = fetch_url_response( + url, headers, timeout=timeout, + cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive, + use_tor=use_tor) + response_time = time.time() + + content = response.read() + read_finish = time.time() + + cleanup_func(response) # release_connection for urllib3 if (response.status == 429 and content.startswith(b'<!DOCTYPE') @@ -185,7 +197,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja elif response.status >= 400: raise FetchError(str(response.status), reason=response.reason, ip=None) - read_finish = time.time() if report_text: print(report_text, ' Latency:', round(response_time - start_time,3), ' Read time:', round(read_finish - response_time,3)) content = decode_content(content, response.getheader('Content-Encoding', default='identity')) @@ -198,8 +209,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja with open(os.path.join(save_dir, debug_name), 'wb') as f: f.write(content) - if return_response: - return content, response return content def head(url, use_tor=False, report_text=None, max_redirects=10): diff --git a/youtube/watch.py b/youtube/watch.py index c1f5e1e..cedf632 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -24,7 +24,7 @@ except FileNotFoundError: def get_video_sources(info): video_sources = [] - if not settings.theater_mode: + if (not settings.theater_mode) or settings.route_tor == 2: max_resolution = 360 else: max_resolution = settings.default_resolution @@ -270,10 +270,11 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): else: info['hls_formats'] = [] - # check for 403 + # check for 403. Unnecessary for tor video routing b/c ip address is same info['invidious_used'] = False info['invidious_reload_button'] = False - if settings.route_tor and info['formats'] and info['formats'][0]['url']: + if (settings.route_tor == 1 + and info['formats'] and info['formats'][0]['url']): try: response = util.head(info['formats'][0]['url'], report_text='Checked for URL access') @@ -408,10 +409,10 @@ def get_watch_page(video_id=None): "author": info['author'], } + # prefix urls, and other post-processing not handled by yt_data_extract for item in info['related_videos']: util.prefix_urls(item) util.add_extra_html_info(item) - if info['playlist']: playlist_id = info['playlist']['id'] for item in info['playlist']['items']: @@ -423,6 +424,11 @@ def get_watch_page(video_id=None): item['url'] += '&index=' + str(item['index']) info['playlist']['author_url'] = util.prefix_url( info['playlist']['author_url']) + # Don't prefix hls_formats for now because the urls inside the manifest + # would need to be prefixed as well. + for fmt in info['formats']: + fmt['url'] = util.prefix_url(fmt['url']) + if settings.gather_googlevideo_domains: with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: |