Add tor video routing

Includes non-tor video routing by default, so no more chances
of the browser leaking headers or user agent to googlevideo
Adjust settings upgrade system to facilitate change to route_tor
setting.
Add some more space on settings page for dropdown settings so does
not overflow due to options with long names.
Closes #7

3 files changed, 34 insertions, 19 deletions

diff --git a/youtube/templates/settings.html b/youtube/templates/settings.html
index 19a2461..5d1df5f 100644
--- a/youtube/templates/settings.html
+++ b/youtube/templates/settings.html
@@ -4,7 +4,7 @@
 {% block style %}
     .settings-form {
         margin: auto;
-        width: 500px;
+        width: 600px;
         margin-top:10px;
         padding: 10px;
         display: block;
diff --git a/youtube/util.py b/youtube/util.py
index b19f91b..77c4fb1 100644
--- a/youtube/util.py
+++ b/youtube/util.py
@@ -119,8 +119,11 @@ def decode_content(content, encoding_header):
             content = gzip.decompress(content)
     return content
 
-def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False, debug_name=None):
+def fetch_url_response(url, headers=(), timeout=15, data=None,
+                       cookiejar_send=None, cookiejar_receive=None,
+                       use_tor=True):
     '''
+    returns response, cleanup_function
     When cookiejar_send is set to a CookieJar object,
      those cookies will be sent in the request (but cookies in response will not be merged into it)
     When cookiejar_receive is set to a CookieJar object,
@@ -147,8 +150,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
         elif not isinstance(data, bytes):
             data = urllib.parse.urlencode(data).encode('ascii')
 
-    start_time = time.time()
-
     if cookiejar_send is not None or cookiejar_receive is not None:     # Use urllib
         req = urllib.request.Request(url, data=data, headers=headers)
 
@@ -160,19 +161,30 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
             opener = urllib.request.build_opener(cookie_processor)
 
         response = opener.open(req, timeout=timeout)
-        response_time = time.time()
-
-
-        content = response.read()
+        cleanup_func = (lambda r: None)
 
     else:           # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them.
         pool = get_pool(use_tor and settings.route_tor)
-
         response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False)
-        response_time = time.time()
+        cleanup_func = (lambda r: r.release_conn())
+
+    return response, cleanup_func
 
-        content = response.read()
-        response.release_conn()
+def fetch_url(url, headers=(), timeout=15, report_text=None, data=None,
+              cookiejar_send=None, cookiejar_receive=None, use_tor=True,
+              debug_name=None):
+    start_time = time.time()
+
+    response, cleanup_func = fetch_url_response(
+        url, headers, timeout=timeout,
+        cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive,
+        use_tor=use_tor)
+    response_time = time.time()
+
+    content = response.read()
+    read_finish = time.time()
+
+    cleanup_func(response)  # release_connection for urllib3
 
     if (response.status == 429
             and content.startswith(b'<!DOCTYPE')
@@ -185,7 +197,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
     elif response.status >= 400:
         raise FetchError(str(response.status), reason=response.reason, ip=None)
 
-    read_finish = time.time()
     if report_text:
         print(report_text, '    Latency:', round(response_time - start_time,3), '    Read time:', round(read_finish - response_time,3))
     content = decode_content(content, response.getheader('Content-Encoding', default='identity'))
@@ -198,8 +209,6 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookieja
         with open(os.path.join(save_dir, debug_name), 'wb') as f:
             f.write(content)
 
-    if return_response:
-        return content, response
     return content
 
 def head(url, use_tor=False, report_text=None, max_redirects=10):
diff --git a/youtube/watch.py b/youtube/watch.py
index c1f5e1e..cedf632 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -24,7 +24,7 @@ except FileNotFoundError:
 
 def get_video_sources(info):
     video_sources = []
-    if not settings.theater_mode:
+    if (not settings.theater_mode) or settings.route_tor == 2:
         max_resolution = 360
     else:
         max_resolution = settings.default_resolution
@@ -270,10 +270,11 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None):
     else:
         info['hls_formats'] = []
 
-    # check for 403
+    # check for 403. Unnecessary for tor video routing b/c ip address is same
     info['invidious_used'] = False
     info['invidious_reload_button'] = False
-    if settings.route_tor and info['formats'] and info['formats'][0]['url']:
+    if (settings.route_tor == 1
+            and info['formats'] and info['formats'][0]['url']):
         try:
             response = util.head(info['formats'][0]['url'],
                 report_text='Checked for URL access')
@@ -408,10 +409,10 @@ def get_watch_page(video_id=None):
         "author":   info['author'],
     }
 
+    # prefix urls, and other post-processing not handled by yt_data_extract
     for item in info['related_videos']:
         util.prefix_urls(item)
         util.add_extra_html_info(item)
-
     if info['playlist']:
         playlist_id = info['playlist']['id']
         for item in info['playlist']['items']:
@@ -423,6 +424,11 @@ def get_watch_page(video_id=None):
                 item['url'] += '&index=' + str(item['index'])
         info['playlist']['author_url'] = util.prefix_url(
             info['playlist']['author_url'])
+    # Don't prefix hls_formats for now because the urls inside the manifest
+    # would need to be prefixed as well.
+    for fmt in info['formats']:
+        fmt['url'] = util.prefix_url(fmt['url'])
+
 
     if settings.gather_googlevideo_domains:
         with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: