aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--server.py59
-rw-r--r--settings.py157
-rw-r--r--youtube/__init__.py17
-rw-r--r--youtube/channel.py42
-rw-r--r--youtube/templates/comments.html10
-rw-r--r--youtube/templates/settings.html2
-rw-r--r--youtube/templates/watch.html2
-rw-r--r--youtube/util.py55
-rw-r--r--youtube/watch.py14
9 files changed, 231 insertions, 127 deletions
diff --git a/server.py b/server.py
index cc59b19..6fb61de 100644
--- a/server.py
+++ b/server.py
@@ -32,24 +32,52 @@ def youtu_be(env, start_response):
env['QUERY_STRING'] += '&v=' + id
yield from yt_app(env, start_response)
-def proxy_site(env, start_response):
+def proxy_site(env, start_response, video=False):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
'Accept': '*/*',
}
+ if 'HTTP_RANGE' in env:
+ headers['Range'] = env['HTTP_RANGE']
+
url = "https://" + env['SERVER_NAME'] + env['PATH_INFO']
if env['QUERY_STRING']:
url += '?' + env['QUERY_STRING']
-
- content, response = util.fetch_url(url, headers, return_response=True)
+ if video and settings.route_tor == 1:
+ response, cleanup_func = util.fetch_url_response(url, headers,
+ use_tor=False,
+ max_redirects=10)
+ elif video:
+ response, cleanup_func = util.fetch_url_response(url, headers,
+ max_redirects=10)
+ else:
+ response, cleanup_func = util.fetch_url_response(url, headers)
headers = response.getheaders()
if isinstance(headers, urllib3._collections.HTTPHeaderDict):
headers = headers.items()
- start_response('200 OK', headers )
- yield content
+ start_response(str(response.status) + ' ' + response.reason, headers)
+ while True:
+ # a bit over 3 seconds of 360p video
+ # we want each TCP packet to transmit in large multiples,
+ # such as 65,536, so we shouldn't read in small chunks
+ # such as 8192 lest that causes the socket library to limit the
+ # TCP window size
+ # Might need fine-tuning, since this gives us 4*65536
+ # The tradeoff is that larger values (such as 6 seconds) only
+ # allows video to buffer in those increments, meaning user must wait
+ # until the entire chunk is downloaded before video starts playing
+ content_part = response.read(32*8192)
+ if not content_part:
+ break
+ yield content_part
+
+ cleanup_func(response)
+
+def proxy_video(env, start_response):
+ yield from proxy_site(env, start_response, video=True)
site_handlers = {
'youtube.com':yt_app,
@@ -57,7 +85,7 @@ site_handlers = {
'ytimg.com': proxy_site,
'yt3.ggpht.com': proxy_site,
'lh3.googleusercontent.com': proxy_site,
-
+ 'googlevideo.com': proxy_video,
}
def split_url(url):
@@ -133,12 +161,15 @@ class FilteredRequestLog:
if not self.filter_re.search(s):
sys.stderr.write(s)
+if __name__ == '__main__':
+ if settings.allow_foreign_addresses:
+ server = WSGIServer(('0.0.0.0', settings.port_number), site_dispatch,
+ log=FilteredRequestLog())
+ else:
+ server = WSGIServer(('127.0.0.1', settings.port_number), site_dispatch,
+ log=FilteredRequestLog())
+ print('Started httpserver on port' , settings.port_number)
+ server.serve_forever()
-if settings.allow_foreign_addresses:
- server = WSGIServer(('0.0.0.0', settings.port_number), site_dispatch,
- log=FilteredRequestLog())
-else:
- server = WSGIServer(('127.0.0.1', settings.port_number), site_dispatch,
- log=FilteredRequestLog())
-print('Started httpserver on port ' , settings.port_number)
-server.serve_forever()
+# for uwsgi, gunicorn, etc.
+application = site_dispatch
diff --git a/settings.py b/settings.py
index d8e8dce..942f79c 100644
--- a/settings.py
+++ b/settings.py
@@ -7,11 +7,24 @@ import collections
import flask
from flask import request
-settings_info = collections.OrderedDict([
+SETTINGS_INFO = collections.OrderedDict([
('route_tor', {
- 'type': bool,
- 'default': False,
+ 'type': int,
+ 'default': 0,
'label': 'Route Tor',
+ 'comment': '''0 - Off
+1 - On, except video
+2 - On, including video (see warnings)''',
+ 'options': [
+ (0, 'Off'),
+ (1, 'On, except video'),
+ (2, 'On, including video (see warnings)'),
+ ],
+ }),
+
+ ('tor_port', {
+ 'type': int,
+ 'default': 9150,
'comment': '',
}),
@@ -148,14 +161,14 @@ For security reasons, enabling this is not recommended.''',
('settings_version', {
'type': int,
- 'default': 2,
+ 'default': 3,
'comment': '''Do not change, remove, or comment out this value, or else your settings may be lost or corrupted''',
'hidden': True,
}),
])
program_directory = os.path.dirname(os.path.realpath(__file__))
-acceptable_targets = settings_info.keys() | {'enable_comments', 'enable_related_videos'}
+acceptable_targets = SETTINGS_INFO.keys() | {'enable_comments', 'enable_related_videos'}
def comment_string(comment):
@@ -164,57 +177,49 @@ def comment_string(comment):
result += '# ' + line + '\n'
return result
-def save_settings(settings):
+def save_settings(settings_dict):
with open(settings_file_path, 'w', encoding='utf-8') as file:
- for setting_name, default_setting_dict in settings_info.items():
- file.write(comment_string(default_setting_dict['comment']) + setting_name + ' = ' + repr(settings[setting_name]) + '\n\n')
+ for setting_name, setting_info in SETTINGS_INFO.items():
+ file.write(comment_string(setting_info['comment']) + setting_name + ' = ' + repr(settings_dict[setting_name]) + '\n\n')
-
-def create_missing_settings_string(current_settings):
- result = ''
- for setting_name, setting_dict in settings_info.items():
- if setting_name not in current_settings:
- result += comment_string(setting_dict['comment']) + setting_name + ' = ' + repr(setting_dict['default']) + '\n\n'
- return result
-
-def create_default_settings_string():
- return settings_to_string({})
-
-def add_missing_settings(settings):
+def add_missing_settings(settings_dict):
result = default_settings()
- result.update(settings)
+ result.update(settings_dict)
return result
def default_settings():
- return {key: setting_info['default'] for key, setting_info in settings_info.items()}
-
-def settings_to_string(settings):
- '''Given a dictionary with the setting names/setting values for the keys/values, outputs a settings file string.
- Fills in missing values from the defaults.'''
- result = ''
- for setting_name, default_setting_dict in settings_info.items():
- if setting_name in settings:
- value = settings[setting_name]
- else:
- value = default_setting_dict['default']
- result += comment_string(default_setting_dict['comment']) + setting_name + ' = ' + repr(value) + '\n\n'
- return result
-
+ return {key: setting_info['default'] for key, setting_info in SETTINGS_INFO.items()}
-def upgrade_to_2(current_settings):
+def upgrade_to_2(settings_dict):
'''Upgrade to settings version 2'''
- new_settings = current_settings.copy()
- if 'enable_comments' in current_settings:
- new_settings['comments_mode'] = int(current_settings['enable_comments'])
+ new_settings = settings_dict.copy()
+ if 'enable_comments' in settings_dict:
+ new_settings['comments_mode'] = int(settings_dict['enable_comments'])
del new_settings['enable_comments']
- if 'enable_related_videos' in current_settings:
- new_settings['related_videos_mode'] = int(current_settings['enable_related_videos'])
+ if 'enable_related_videos' in settings_dict:
+ new_settings['related_videos_mode'] = int(settings_dict['enable_related_videos'])
del new_settings['enable_related_videos']
+ new_settings['settings_version'] = 2
return new_settings
+def upgrade_to_3(settings_dict):
+ new_settings = settings_dict.copy()
+ if 'route_tor' in settings_dict:
+ new_settings['route_tor'] = int(settings_dict['route_tor'])
+ new_settings['settings_version'] = 3
+ return new_settings
+
+upgrade_functions = {
+ 1: upgrade_to_2,
+ 2: upgrade_to_3,
+}
+
def log_ignored_line(line_number, message):
print("WARNING: Ignoring settings.txt line " + str(node.lineno) + " (" + message + ")")
+
+
+
if os.path.isfile("settings.txt"):
print("Running in portable mode")
settings_dir = os.path.normpath('./')
@@ -232,15 +237,15 @@ try:
with open(settings_file_path, 'r', encoding='utf-8') as file:
settings_text = file.read()
except FileNotFoundError:
- settings = default_settings()
- save_settings(settings)
+ current_settings_dict = default_settings()
+ save_settings(current_settings_dict)
else:
if re.fullmatch(r'\s*', settings_text): # blank file
- settings = default_settings()
- save_settings(settings)
+ current_settings_dict = default_settings()
+ save_settings(current_settings_dict)
else:
# parse settings in a safe way, without exec
- settings = {}
+ current_settings_dict = {}
attributes = {
ast.Constant: 'value',
ast.NameConstant: 'value',
@@ -270,21 +275,27 @@ else:
log_ignored_line(node.lineno, "only literals allowed for values")
continue
- settings[target.id] = node.value.__getattribute__(attributes[type(node.value)])
-
+ current_settings_dict[target.id] = node.value.__getattribute__(attributes[type(node.value)])
- if 'settings_version' not in settings:
- print('Upgrading settings.txt')
- settings = add_missing_settings(upgrade_to_2(settings))
- save_settings(settings)
+ # upgrades
+ latest_version = SETTINGS_INFO['settings_version']['default']
+ while current_settings_dict.get('settings_version',1) < latest_version:
+ current_version = current_settings_dict.get('settings_version', 1)
+ print('Upgrading settings.txt to version', current_version+1)
+ upgrade_func = upgrade_functions[current_version]
+ # Must add missing settings here rather than below because
+ # save_settings needs all settings to be present
+ current_settings_dict = add_missing_settings(
+ upgrade_func(current_settings_dict))
+ save_settings(current_settings_dict)
# some settings not in the file, add those missing settings to the file
- elif not settings.keys() >= settings_info.keys():
+ if not current_settings_dict.keys() >= SETTINGS_INFO.keys():
print('Adding missing settings to settings.txt')
- settings = add_missing_settings(settings)
- save_settings(settings)
+ current_settings_dict = add_missing_settings(current_settings_dict)
+ save_settings(current_settings_dict)
-locals().update(settings)
+globals().update(current_settings_dict)
@@ -294,6 +305,9 @@ if route_tor:
else:
print("Tor routing is OFF - your Youtube activity is NOT anonymous")
+
+
+
hooks = {}
def add_setting_changed_hook(setting, func):
'''Called right before new settings take effect'''
@@ -306,43 +320,34 @@ def add_setting_changed_hook(setting, func):
def settings_page():
if request.method == 'GET':
return flask.render_template('settings.html',
- settings = [(setting_name, setting_info, settings[setting_name]) for setting_name, setting_info in settings_info.items()]
+ settings = [(setting_name, setting_info, current_settings_dict[setting_name]) for setting_name, setting_info in SETTINGS_INFO.items()]
)
elif request.method == 'POST':
for key, value in request.values.items():
- if key in settings_info:
- if settings_info[key]['type'] is bool and value == 'on':
- settings[key] = True
+ if key in SETTINGS_INFO:
+ if SETTINGS_INFO[key]['type'] is bool and value == 'on':
+ current_settings_dict[key] = True
else:
- settings[key] = settings_info[key]['type'](value)
+ current_settings_dict[key] = SETTINGS_INFO[key]['type'](value)
else:
flask.abort(400)
# need this bullshit because browsers don't send anything when an input is unchecked
- expected_inputs = {setting_name for setting_name, setting_info in settings_info.items() if not settings_info[setting_name].get('hidden', False)}
+ expected_inputs = {setting_name for setting_name, setting_info in SETTINGS_INFO.items() if not SETTINGS_INFO[setting_name].get('hidden', False)}
missing_inputs = expected_inputs - set(request.values.keys())
for setting_name in missing_inputs:
- assert settings_info[setting_name]['type'] is bool, missing_inputs
- settings[setting_name] = False
+ assert SETTINGS_INFO[setting_name]['type'] is bool, missing_inputs
+ current_settings_dict[setting_name] = False
# call setting hooks
- for setting_name, value in settings.items():
+ for setting_name, value in current_settings_dict.items():
old_value = globals()[setting_name]
if value != old_value and setting_name in hooks:
for func in hooks[setting_name]:
func(old_value, value)
- globals().update(settings)
- save_settings(settings)
+ globals().update(current_settings_dict)
+ save_settings(current_settings_dict)
return flask.redirect(util.URL_ORIGIN + '/settings', 303)
else:
flask.abort(400)
-
-
-
-
-
-
-
-
-
diff --git a/youtube/__init__.py b/youtube/__init__.py
index 8675c4b..6c2ec48 100644
--- a/youtube/__init__.py
+++ b/youtube/__init__.py
@@ -2,6 +2,7 @@ from youtube import util
import flask
import settings
import traceback
+import re
from sys import exc_info
yt_app = flask.Flask(__name__)
yt_app.url_map.strict_slashes = False
@@ -34,6 +35,22 @@ def commatize(num):
num = int(num)
return '{:,}'.format(num)
+def timestamp_replacement(match):
+ time_seconds = 0
+ for part in match.group(0).split(':'):
+ time_seconds = 60*time_seconds + int(part)
+ return (
+ '<a href="#" onclick="document.querySelector(\'video\').currentTime='
+ + str(time_seconds)
+ + '">' + match.group(0)
+ + '</a>'
+ )
+
+TIMESTAMP_RE = re.compile(r'\b(\d?\d:)?\d?\d:\d\d\b')
+@yt_app.template_filter('timestamps')
+def timestamps(text):
+ return TIMESTAMP_RE.sub(timestamp_replacement, text)
+
@yt_app.errorhandler(500)
def error_page(e):
if (exc_info()[0] == util.FetchError
diff --git a/youtube/channel.py b/youtube/channel.py
index ba1a4d2..ad6db5b 100644
--- a/youtube/channel.py
+++ b/youtube/channel.py
@@ -44,7 +44,28 @@ generic_cookie = (('Cookie', 'VISITOR_INFO1_LIVE=ST1Ti53r4fU'),)
# view:
# grid: 0 or 1
# list: 2
-def channel_ctoken_desktop(channel_id, page, sort, tab, view=1):
+def channel_ctoken_v3(channel_id, page, sort, tab, view=1):
+ # page > 1 doesn't work when sorting by oldest
+ offset = 30*(int(page) - 1)
+ page_token = proto.string(61, proto.unpadded_b64encode(
+ proto.string(1, proto.unpadded_b64encode(proto.uint(1,offset)))
+ ))
+
+ tab = proto.string(2, tab )
+ sort = proto.uint(3, int(sort))
+
+ shelf_view = proto.uint(4, 0)
+ view = proto.uint(6, int(view))
+ continuation_info = proto.string(3,
+ proto.percent_b64encode(tab + sort + shelf_view + view + page_token)
+ )
+
+ channel_id = proto.string(2, channel_id )
+ pointless_nest = proto.string(80226972, channel_id + continuation_info)
+
+ return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
+
+def channel_ctoken_v2(channel_id, page, sort, tab, view=1):
# see https://github.com/iv-org/invidious/issues/1319#issuecomment-671732646
# page > 1 doesn't work when sorting by oldest
offset = 30*(int(page) - 1)
@@ -74,14 +95,14 @@ def channel_ctoken_desktop(channel_id, page, sort, tab, view=1):
return base64.urlsafe_b64encode(pointless_nest).decode('ascii')
-def channel_ctoken_mobile(channel_id, page, sort, tab, view=1):
+def channel_ctoken_v1(channel_id, page, sort, tab, view=1):
tab = proto.string(2, tab )
sort = proto.uint(3, int(sort))
page = proto.string(15, str(page) )
# example with shelves in videos tab: https://www.youtube.com/channel/UCNL1ZadSjHpjm4q9j2sVtOA/videos
shelf_view = proto.uint(4, 0)
view = proto.uint(6, int(view))
- continuation_info = proto.string( 3, proto.percent_b64encode(tab + view + sort + shelf_view + page) )
+ continuation_info = proto.string(3, proto.percent_b64encode(tab + view + sort + shelf_view + page + proto.uint(23, 0)) )
channel_id = proto.string(2, channel_id )
pointless_nest = proto.string(80226972, channel_id + continuation_info)
@@ -91,15 +112,16 @@ def channel_ctoken_mobile(channel_id, page, sort, tab, view=1):
def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1, print_status=True):
message = 'Got channel tab' if print_status else None
- if int(sort) == 2 and int(page) > 1: # use mobile endpoint
- ctoken = channel_ctoken_mobile(channel_id, page, sort, tab, view)
+ if int(sort) == 2 and int(page) > 1:
+ ctoken = channel_ctoken_v1(channel_id, page, sort, tab, view)
ctoken = ctoken.replace('=', '%3D')
- url = ('https://m.youtube.com/channel/' + channel_id + '/' + tab
- + '?ctoken=' + ctoken + '&pbj=1')
- content = util.fetch_url(url, headers_mobile + real_cookie,
+ url = ('https://www.youtube.com/channel/' + channel_id + '/' + tab
+ + '?action_continuation=1&continuation=' + ctoken
+ + '&pbj=1')
+ content = util.fetch_url(url, headers_desktop + real_cookie,
debug_name='channel_tab', report_text=message)
- else: # use desktop endpoint
- ctoken = channel_ctoken_desktop(channel_id, page, sort, tab, view)
+ else:
+ ctoken = channel_ctoken_v3(channel_id, page, sort, tab, view)
ctoken = ctoken.replace('=', '%3D')
url = 'https://www.youtube.com/browse_ajax?ctoken=' + ctoken
content = util.fetch_url(url,
diff --git a/youtube/templates/comments.html b/youtube/templates/comments.html
index 396852a..f2cdf65 100644
--- a/youtube/templates/comments.html
+++ b/youtube/templates/comments.html
@@ -1,6 +1,6 @@
{% import "common_elements.html" as common_elements %}
-{% macro render_comment(comment, include_avatar) %}
+{% macro render_comment(comment, include_avatar, timestamp_links=False) %}
<div class="comment-container">
<div class="comment">
<a class="author-avatar" href="{{ comment['author_url'] }}" title="{{ comment['author'] }}">
@@ -14,7 +14,11 @@
<a class="permalink" href="{{ comment['permalink'] }}" title="permalink">
<time datetime="">{{ comment['time_published'] }}</time>
</a>
- <span class="text">{{ common_elements.text_runs(comment['text']) }}</span>
+ {% if timestamp_links %}
+ <span class="text">{{ common_elements.text_runs(comment['text'])|timestamps|safe }}</span>
+ {% else %}
+ <span class="text">{{ common_elements.text_runs(comment['text']) }}</span>
+ {% endif %}
<span class="likes">{{ comment['likes_text'] if comment['like_count'] else ''}}</span>
<div class="bottom-row">
@@ -36,7 +40,7 @@
</div>
<div class="comments">
{% for comment in comments_info['comments'] %}
- {{ render_comment(comment, comments_info['include_avatars']) }}
+ {{ render_comment(comment, comments_info['include_avatars'], True) }}
{% endfor %}
</div>
{% if 'more_comments_url' is in comments_info %}
diff --git a/youtube/templates/settings.html b/youtube/templates/settings.html
index 19a2461..5d1df5f 100644
--- a/youtube/templates/settings.html
+++ b/youtube/templates/settings.html
@@ -4,7 +4,7 @@
{% block style %}
.settings-form {
margin: auto;
- width: 500px;
+ width: 600px;
margin-top:10px;
padding: 10px;
display: block;
diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html
index 8264eb8..e3c6fa0 100644
--- a/youtube/templates/watch.html
+++ b/youtube/templates/watch.html
@@ -413,7 +413,7 @@ Reload without invidious (for usage of new identity button).</a>
<input class="checkbox" name="video_info_list" value="{{ video_info }}" form="playlist-edit" type="checkbox">
- <span class="description">{{ common_elements.text_runs(description)|urlize }}</span>
+ <span class="description">{{ common_elements.text_runs(description)|escape|urlize|timestamps|safe }}</span>
<div class="music-list">
{% if music_list.__len__() != 0 %}
<hr>
diff --git a/youtube/util.py b/youtube/util.py
index b19f91b..3c32ddb 100644
--- a/youtube/util.py
+++ b/youtube/util.py
@@ -54,7 +54,7 @@ URL_ORIGIN = "/https://www.youtube.com"
connection_pool = urllib3.PoolManager(cert_reqs = 'CERT_REQUIRED')
old_tor_connection_pool = None
-tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:9150/', cert_reqs = 'CERT_REQUIRED')
+tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:' + str(settings.tor_port) + '/', cert_reqs = 'CERT_REQUIRED')
tor_pool_refresh_time = time.monotonic() # prevent problems due to clock changes
@@ -74,7 +74,7 @@ def get_pool(use_tor):
# Keep a reference for 5 min to avoid it getting garbage collected while sockets still in use
old_tor_connection_pool = tor_connection_pool
- tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:9150/', cert_reqs = 'CERT_REQUIRED')
+ tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:' + str(settings.tor_port) + '/', cert_reqs = 'CERT_REQUIRED')
tor_pool_refresh_time = current_time
return tor_connection_pool
@@ -119,8 +119,11 @@ def decode_content(content, encoding_header):
content = gzip.decompress(content)
return content
-def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False, debug_name=None):
+def fetch_url_response(url, headers=(), timeout=15, data=None,
+ cookiejar_send=None, cookiejar_receive=None,
+ use_tor=True, max_redirects=None):
'''
+ returns response, cleanup_function
When cookiejar_send is set to a CookieJar object,
those cookies will be sent in the request (but cookies in response will not be merged into it)
When cookiejar_receive is set to a CookieJar object,
@@ -147,32 +150,51 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
elif not isinstance(data, bytes):
data = urllib.parse.urlencode(data).encode('ascii')
- start_time = time.time()
-
if cookiejar_send is not None or cookiejar_receive is not None: # Use urllib
req = urllib.request.Request(url, data=data, headers=headers)
cookie_processor = HTTPAsymmetricCookieProcessor(cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive)
if use_tor and settings.route_tor:
- opener = urllib.request.build_opener(sockshandler.SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150), cookie_processor)
+ opener = urllib.request.build_opener(sockshandler.SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", settings.tor_port), cookie_processor)
else:
opener = urllib.request.build_opener(cookie_processor)
response = opener.open(req, timeout=timeout)
- response_time = time.time()
-
-
- content = response.read()
+ cleanup_func = (lambda r: None)
else: # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them.
+ # default: Retry.DEFAULT = Retry(3)
+ # (in connectionpool.py in urllib3)
+ # According to the documentation for urlopen, a redirect counts as a
+ # retry. So there are 3 redirects max by default.
+ if max_redirects:
+ retries = urllib3.Retry(3+max_redirects, redirect=max_redirects)
+ else:
+ retries = urllib3.Retry(3)
pool = get_pool(use_tor and settings.route_tor)
+ response = pool.request(method, url, headers=headers,
+ timeout=timeout, preload_content=False,
+ decode_content=False, retries=retries)
+ cleanup_func = (lambda r: r.release_conn())
- response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False)
- response_time = time.time()
+ return response, cleanup_func
- content = response.read()
- response.release_conn()
+def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
+ cookiejar_send=None, cookiejar_receive=None, use_tor=True,
+ debug_name=None):
+ start_time = time.time()
+
+ response, cleanup_func = fetch_url_response(
+ url, headers, timeout=timeout,
+ cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive,
+ use_tor=use_tor)
+ response_time = time.time()
+
+ content = response.read()
+ read_finish = time.time()
+
+ cleanup_func(response) # release_connection for urllib3
if (response.status == 429
and content.startswith(b'<!DOCTYPE')
@@ -185,7 +207,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
elif response.status >= 400:
raise FetchError(str(response.status), reason=response.reason, ip=None)
- read_finish = time.time()
if report_text:
print(report_text, ' Latency:', round(response_time - start_time,3), ' Read time:', round(read_finish - response_time,3))
content = decode_content(content, response.getheader('Content-Encoding', default='identity'))
@@ -198,8 +219,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
with open(os.path.join(save_dir, debug_name), 'wb') as f:
f.write(content)
- if return_response:
- return content, response
return content
def head(url, use_tor=False, report_text=None, max_redirects=10):
@@ -209,7 +228,7 @@ def head(url, use_tor=False, report_text=None, max_redirects=10):
# default: Retry.DEFAULT = Retry(3)
# (in connectionpool.py in urllib3)
# According to the documentation for urlopen, a redirect counts as a retry
- # by default. So there are 3 redirects max by default. Let's change that
+ # So there are 3 redirects max by default. Let's change that
# to 10 since googlevideo redirects a lot.
retries = urllib3.Retry(3+max_redirects, redirect=max_redirects,
raise_on_redirect=False)
diff --git a/youtube/watch.py b/youtube/watch.py
index c1f5e1e..cedf632 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -24,7 +24,7 @@ except FileNotFoundError:
def get_video_sources(info):
video_sources = []
- if not settings.theater_mode:
+ if (not settings.theater_mode) or settings.route_tor == 2:
max_resolution = 360
else:
max_resolution = settings.default_resolution
@@ -270,10 +270,11 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
else:
info['hls_formats'] = []
- # check for 403
+ # check for 403. Unnecessary for tor video routing b/c ip address is same
info['invidious_used'] = False
info['invidious_reload_button'] = False
- if settings.route_tor and info['formats'] and info['formats'][0]['url']:
+ if (settings.route_tor == 1
+ and info['formats'] and info['formats'][0]['url']):
try:
response = util.head(info['formats'][0]['url'],
report_text='Checked for URL access')
@@ -408,10 +409,10 @@ def get_watch_page(video_id=None):
"author": info['author'],
}
+ # prefix urls, and other post-processing not handled by yt_data_extract
for item in info['related_videos']:
util.prefix_urls(item)
util.add_extra_html_info(item)
-
if info['playlist']:
playlist_id = info['playlist']['id']
for item in info['playlist']['items']:
@@ -423,6 +424,11 @@ def get_watch_page(video_id=None):
item['url'] += '&index=' + str(item['index'])
info['playlist']['author_url'] = util.prefix_url(
info['playlist']['author_url'])
+ # Don't prefix hls_formats for now because the urls inside the manifest
+ # would need to be prefixed as well.
+ for fmt in info['formats']:
+ fmt['url'] = util.prefix_url(fmt['url'])
+
if settings.gather_googlevideo_domains:
with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f:
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-15 04:18:46 +0000