Bring up to date with master

14 files changed, 612 insertions, 533 deletions

diff --git a/youtube/accounts.py b/youtube/accounts.py
index bde9852..375bf2a 100644
--- a/youtube/accounts.py
+++ b/youtube/accounts.py
@@ -1,10 +1,10 @@
 # Contains functions having to do with logging in
+from youtube import util, html_common
+import settings
 
 import urllib
 import json
-from youtube import common
 import re
-import settings
 import http.cookiejar
 import io
 import os
@@ -106,7 +106,7 @@ def get_account_login_page(env, start_response):
     '''
 
     page = '''
-    <form action="''' + common.URL_ORIGIN + '''/login" method="POST">
+    <form action="''' + util.URL_ORIGIN + '''/login" method="POST">
         <div class="form-field">
             <label for="username">Username:</label>
             <input type="text" id="username" name="username">
@@ -130,10 +130,10 @@ Using Tor to log in should only be done if the account was created using a proxy
     </div>
     '''
 
-    return common.yt_basic_template.substitute(
+    return html_common.yt_basic_template.substitute(
         page_title = "Login",
         style = style,
-        header = common.get_header(),
+        header = html_common.get_header(),
         page = page,
     ).encode('utf-8')
 
@@ -229,7 +229,7 @@ def _login(username, password, cookiejar, use_tor):
     Taken from youtube-dl
     """
 
-    login_page = common.fetch_url(_LOGIN_URL, yt_dl_headers, report_text='Downloaded login page', cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
+    login_page = util.fetch_url(_LOGIN_URL, yt_dl_headers, report_text='Downloaded login page', cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
     '''with open('debug/login_page', 'w', encoding='utf-8') as f:
         f.write(login_page)'''
     #print(cookiejar.as_lwp_str())
@@ -255,7 +255,7 @@ def _login(username, password, cookiejar, use_tor):
             'Google-Accounts-XSRF': 1,
         }
         headers.update(yt_dl_headers)
-        result = common.fetch_url(url, headers, report_text=note, data=data, cookiejar_send=cookiejar, cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
+        result = util.fetch_url(url, headers, report_text=note, data=data, cookiejar_send=cookiejar, cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
         #print(cookiejar.as_lwp_str())
         '''with open('debug/' + note, 'w', encoding='utf-8') as f:
             f.write(result)'''
@@ -387,7 +387,7 @@ def _login(username, password, cookiejar, use_tor):
         return False
 
     try:
-        check_cookie_results = common.fetch_url(check_cookie_url, headers=yt_dl_headers, report_text="Checked cookie", cookiejar_send=cookiejar, cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
+        check_cookie_results = util.fetch_url(check_cookie_url, headers=yt_dl_headers, report_text="Checked cookie", cookiejar_send=cookiejar, cookiejar_receive=cookiejar, use_tor=use_tor).decode('utf-8')
     except (urllib.error.URLError, compat_http_client.HTTPException, socket.error) as err:
         return False
 
@@ -398,7 +398,7 @@ def _login(username, password, cookiejar, use_tor):
         warn('Unable to log in')
         return False
 
-    select_site_page = common.fetch_url('https://m.youtube.com/select_site', headers=common.mobile_ua, report_text="Retrieved page for channel id", cookiejar_send=cookiejar, use_tor=use_tor).decode('utf-8')
+    select_site_page = util.fetch_url('https://m.youtube.com/select_site', headers=util.mobile_ua, report_text="Retrieved page for channel id", cookiejar_send=cookiejar, use_tor=use_tor).decode('utf-8')
     match = _CHANNEL_ID_RE.search(select_site_page)
     if match is None:
         warn('Failed to find channel id')
diff --git a/youtube/channel.py b/youtube/channel.py
index c83d7d1..55316e2 100644
--- a/youtube/channel.py
+++ b/youtube/channel.py
@@ -1,6 +1,6 @@
 import base64
-import youtube.common as common
-from youtube.common import default_multi_get, URL_ORIGIN, get_thumbnail_url, video_id
+from youtube import util, yt_data_extract, html_common
+
 import http_errors
 import urllib
 import json
@@ -91,7 +91,7 @@ def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1):
     url = "https://www.youtube.com/browse_ajax?ctoken=" + ctoken
 
     print("Sending channel tab ajax request")
-    content = common.fetch_url(url, common.desktop_ua + headers_1)
+    content = util.fetch_url(url, util.desktop_ua + headers_1)
     print("Finished recieving channel tab response")
 
     '''with open('debug/channel_debug', 'wb') as f:
@@ -110,7 +110,7 @@ def get_number_of_videos(channel_id):
 
     # Sometimes retrieving playlist info fails with 403 for no discernable reason
     try:
-        response = common.fetch_url(url, common.mobile_ua + headers_pbj)
+        response = util.fetch_url(url, util.mobile_ua + headers_pbj)
     except urllib.error.HTTPError as e:
         if e.code != 403:
             raise
@@ -133,20 +133,20 @@ def get_channel_id(username):
     # method that gives the smallest possible response at ~10 kb
     # needs to be as fast as possible
     url = 'https://m.youtube.com/user/' + username + '/about?ajax=1&disable_polymer=true'
-    response = common.fetch_url(url, common.mobile_ua + headers_1).decode('utf-8')
+    response = util.fetch_url(url, util.mobile_ua + headers_1).decode('utf-8')
     return re.search(r'"channel_id":\s*"([a-zA-Z0-9_-]*)"', response).group(1)
 
 def grid_items_html(items, additional_info={}):
     result = '''            <nav class="item-grid">\n'''
     for item in items:
-        result += common.renderer_html(item, additional_info)
+        result += html_common.renderer_html(item, additional_info)
     result += '''\n</nav>'''
     return result
 
 def list_items_html(items, additional_info={}):
     result = '''                <nav class="item-list">'''
     for item in items:
-        result += common.renderer_html(item, additional_info)
+        result += html_common.renderer_html(item, additional_info)
     result += '''\n</nav>'''
     return result
 
@@ -168,11 +168,11 @@ def channel_tabs_html(channel_id, current_tab, search_box_value=''):
             )
         else:
             result += channel_tab_template.substitute(
-                href_attribute = ' href="' + URL_ORIGIN + '/channel/' + channel_id + '/' + tab_name.lower() + '"',
+                href_attribute = ' href="' + util.URL_ORIGIN + '/channel/' + channel_id + '/' + tab_name.lower() + '"',
                 tab_name = tab_name,
             )
     result += channel_search_template.substitute(
-        action = URL_ORIGIN + "/channel/" + channel_id + "/search",
+        action = util.URL_ORIGIN + "/channel/" + channel_id + "/search",
         search_box_value = html.escape(search_box_value),
     )
     return result
@@ -192,7 +192,7 @@ def channel_sort_buttons_html(channel_id, tab, current_sort):
             )
         else:
             result += channel_sort_button_template.substitute(
-                href_attribute=' href="' + URL_ORIGIN + '/channel/' + channel_id + '/' + tab + '?sort=' + sort_number + '"',
+                href_attribute=' href="' + util.URL_ORIGIN + '/channel/' + channel_id + '/' + tab + '?sort=' + sort_number + '"',
                 text = 'Sort by ' + sort_name
             )
     return result
@@ -246,7 +246,7 @@ def channel_videos_html(polymer_json, current_page=1, current_sort=3, number_of_
     items_html = grid_items_html(items, {'author': microformat['title']})
     
     return yt_channel_items_template.substitute(
-        header              = common.get_header(),
+        header              = html_common.get_header(),
         channel_title       = microformat['title'],
         channel_id          = channel_id,
         channel_tabs        = channel_tabs_html(channel_id, 'Videos'),
@@ -254,7 +254,7 @@ def channel_videos_html(polymer_json, current_page=1, current_sort=3, number_of_
         avatar              = '/' + microformat['thumbnail']['thumbnails'][0]['url'],
         page_title          = microformat['title'] + ' - Channel',
         items               = items_html,
-        page_buttons        = common.page_buttons_html(current_page, math.ceil(number_of_videos/30), URL_ORIGIN + "/channel/" + channel_id + "/videos", current_query_string),
+        page_buttons        = html_common.page_buttons_html(current_page, math.ceil(number_of_videos/30), util.URL_ORIGIN + "/channel/" + channel_id + "/videos", current_query_string),
         number_of_results   = '{:,}'.format(number_of_videos) + " videos",
     )
 
@@ -268,7 +268,7 @@ def channel_playlists_html(polymer_json, current_sort=3):
     items_html = grid_items_html(items, {'author': microformat['title']})
     
     return yt_channel_items_template.substitute(
-        header              = common.get_header(),
+        header              = html_common.get_header(),
         channel_title       = microformat['title'],
         channel_id          = channel_id,
         channel_tabs        = channel_tabs_html(channel_id, 'Playlists'),
@@ -312,25 +312,25 @@ def channel_about_page(polymer_json):
 
         channel_links += channel_link_template.substitute(
             url     = html.escape(url),
-            text    = common.get_plain_text(link_json['title']),
+            text    = yt_data_extract.get_plain_text(link_json['title']),
         )
 
     stats = ''
     for stat_name in ('subscriberCountText', 'joinedDateText', 'viewCountText', 'country'):
         try:
-            stat_value = common.get_plain_text(channel_metadata[stat_name])
+            stat_value = yt_data_extract.get_plain_text(channel_metadata[stat_name])
         except KeyError:
             continue
         else:
             stats += stat_template.substitute(stat_value=stat_value)
     try:
-        description = common.format_text_runs(common.get_formatted_text(channel_metadata['description']))
+        description = yt_data_extract.format_text_runs(yt_data_extract.get_formatted_text(channel_metadata['description']))
     except KeyError:
         description = ''
     return yt_channel_about_template.substitute(
-        header              = common.get_header(),
-        page_title          = common.get_plain_text(channel_metadata['title']) + ' - About',
-        channel_title       = common.get_plain_text(channel_metadata['title']),
+        header              = html_common.get_header(),
+        page_title          = yt_data_extract.get_plain_text(channel_metadata['title']) + ' - About',
+        channel_title       = yt_data_extract.get_plain_text(channel_metadata['title']),
         avatar              = html.escape(avatar),
         description         = description,
         links               = channel_links,
@@ -354,14 +354,14 @@ def channel_search_page(polymer_json, query, current_page=1, number_of_videos =
     items_html = list_items_html(items)
 
     return yt_channel_items_template.substitute(
-        header              = common.get_header(),
+        header              = html_common.get_header(),
         channel_title       = html.escape(microformat['title']),
         channel_id          = channel_id,
         channel_tabs        = channel_tabs_html(channel_id, '', query),
         avatar              = '/' + microformat['thumbnail']['thumbnails'][0]['url'],
         page_title          = html.escape(query + ' - Channel search'),
         items               = items_html,
-        page_buttons        = common.page_buttons_html(current_page, math.ceil(number_of_videos/29), URL_ORIGIN + "/channel/" + channel_id + "/search", current_query_string),
+        page_buttons        = html_common.page_buttons_html(current_page, math.ceil(number_of_videos/29), util.URL_ORIGIN + "/channel/" + channel_id + "/search", current_query_string),
         number_of_results   = '',
         sort_buttons        = '',
     )
@@ -371,7 +371,7 @@ def get_channel_search_json(channel_id, query, page):
     ctoken = proto.string(2, channel_id) + proto.string(3, params) + proto.string(11, query)
     ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii')
 
-    polymer_json = common.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, common.desktop_ua + headers_1)
+    polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, util.desktop_ua + headers_1)
     '''with open('debug/channel_search_debug', 'wb') as f:
         f.write(polymer_json)'''
     polymer_json = json.loads(polymer_json)
@@ -388,10 +388,10 @@ def get_channel_page(env, start_response):
         tab = 'videos'
     
     parameters = env['parameters']
-    page_number = int(common.default_multi_get(parameters, 'page', 0, default='1'))
-    sort = common.default_multi_get(parameters, 'sort', 0, default='3')
-    view = common.default_multi_get(parameters, 'view', 0, default='1')
-    query = common.default_multi_get(parameters, 'query', 0, default='')
+    page_number = int(util.default_multi_get(parameters, 'page', 0, default='1'))
+    sort = util.default_multi_get(parameters, 'sort', 0, default='3')
+    view = util.default_multi_get(parameters, 'view', 0, default='1')
+    query = util.default_multi_get(parameters, 'query', 0, default='')
 
     if tab == 'videos':
         tasks = (
@@ -403,11 +403,11 @@ def get_channel_page(env, start_response):
 
         result = channel_videos_html(polymer_json, page_number, sort, number_of_videos, env['QUERY_STRING'])
     elif tab == 'about':
-        polymer_json = common.fetch_url('https://www.youtube.com/channel/' + channel_id + '/about?pbj=1', common.desktop_ua + headers_1)
+        polymer_json = util.fetch_url('https://www.youtube.com/channel/' + channel_id + '/about?pbj=1', util.desktop_ua + headers_1)
         polymer_json = json.loads(polymer_json)
         result = channel_about_page(polymer_json)
     elif tab == 'playlists':
-        polymer_json = common.fetch_url('https://www.youtube.com/channel/' + channel_id + '/playlists?pbj=1&view=1&sort=' + playlist_sort_codes[sort], common.desktop_ua + headers_1)
+        polymer_json = util.fetch_url('https://www.youtube.com/channel/' + channel_id + '/playlists?pbj=1&view=1&sort=' + playlist_sort_codes[sort], util.desktop_ua + headers_1)
         '''with open('debug/channel_playlists_debug', 'wb') as f:
             f.write(polymer_json)'''
         polymer_json = json.loads(polymer_json)
@@ -447,22 +447,22 @@ def get_channel_page_general_url(env, start_response):
         return b'Invalid channel url'
 
     if page == 'videos':
-        polymer_json = common.fetch_url(base_url + '/videos?pbj=1&view=0', common.desktop_ua + headers_1)
+        polymer_json = util.fetch_url(base_url + '/videos?pbj=1&view=0', util.desktop_ua + headers_1)
         '''with open('debug/user_page_videos', 'wb') as f:
             f.write(polymer_json)'''
         polymer_json = json.loads(polymer_json)
         result = channel_videos_html(polymer_json)
     elif page == 'about':
-        polymer_json = common.fetch_url(base_url + '/about?pbj=1', common.desktop_ua + headers_1)
+        polymer_json = util.fetch_url(base_url + '/about?pbj=1', util.desktop_ua + headers_1)
         polymer_json = json.loads(polymer_json)
         result = channel_about_page(polymer_json)
     elif page == 'playlists':
-        polymer_json = common.fetch_url(base_url+ '/playlists?pbj=1&view=1', common.desktop_ua + headers_1)
+        polymer_json = util.fetch_url(base_url+ '/playlists?pbj=1&view=1', util.desktop_ua + headers_1)
         polymer_json = json.loads(polymer_json)
         result = channel_playlists_html(polymer_json)
     elif page == 'search':
         raise NotImplementedError()
-        '''polymer_json = common.fetch_url('https://www.youtube.com/user' + username +  '/search?pbj=1&' + query_string, common.desktop_ua + headers_1)
+        '''polymer_json = util.fetch_url('https://www.youtube.com/user' + username +  '/search?pbj=1&' + query_string, util.desktop_ua + headers_1)
         polymer_json = json.loads(polymer_json)
         return channel_search_page('''
     else:
diff --git a/youtube/comments.py b/youtube/comments.py
index 10209e7..94b086e 100644
--- a/youtube/comments.py
+++ b/youtube/comments.py
@@ -1,13 +1,14 @@
+from youtube import proto, util, html_common, yt_data_extract, accounts
+import settings
+
 import json
-from youtube import proto, common, accounts
 import base64
-from youtube.common import uppercase_escape, default_multi_get, format_text_runs, URL_ORIGIN, fetch_url
 from string import Template
 import urllib.request
 import urllib
 import html
-import settings
 import re
+
 comment_area_template = Template('''
 <section class="comment-area">
 $video-metadata
@@ -130,7 +131,7 @@ def request_comments(ctoken, replies=False):
     url = base_url + ctoken.replace("=", "%3D") + "&pbj=1"
 
     for i in range(0,8):    # don't retry more than 8 times
-        content = fetch_url(url, headers=mobile_headers, report_text="Retrieved comments")
+        content = util.fetch_url(url, headers=mobile_headers, report_text="Retrieved comments")
         if content[0:4] == b")]}'":             # random closing characters included at beginning of response for some reason
             content = content[4:]
         elif content[0:10] == b'\n<!DOCTYPE':   # occasionally returns html instead of json for no reason
@@ -151,10 +152,10 @@ def single_comment_ctoken(video_id, comment_id):
 
 def parse_comments_ajax(content, replies=False):
     try:
-        content = json.loads(uppercase_escape(content.decode('utf-8')))
+        content = json.loads(util.uppercase_escape(content.decode('utf-8')))
         #print(content)
         comments_raw = content['content']['continuation_contents']['contents']
-        ctoken = default_multi_get(content, 'content', 'continuation_contents', 'continuations', 0, 'continuation', default='')
+        ctoken = util.default_multi_get(content, 'content', 'continuation_contents', 'continuations', 0, 'continuation', default='')
         
         comments = []
         for comment_raw in comments_raw:
@@ -163,7 +164,7 @@ def parse_comments_ajax(content, replies=False):
                 if comment_raw['replies'] is not None:
                     reply_ctoken = comment_raw['replies']['continuations'][0]['continuation']
                     comment_id, video_id = get_ids(reply_ctoken)
-                    replies_url = URL_ORIGIN + '/comments?parent_id=' + comment_id + "&video_id=" + video_id
+                    replies_url = util.URL_ORIGIN + '/comments?parent_id=' + comment_id + "&video_id=" + video_id
                 comment_raw = comment_raw['comment']
             comment = {
             'author': comment_raw['author']['runs'][0]['text'],
@@ -189,7 +190,7 @@ reply_count_regex = re.compile(r'(\d+)')
 def parse_comments_polymer(content, replies=False):
     try:
         video_title = ''
-        content = json.loads(uppercase_escape(content.decode('utf-8')))
+        content = json.loads(util.uppercase_escape(content.decode('utf-8')))
         url = content[1]['url']
         ctoken = urllib.parse.parse_qs(url[url.find('?')+1:])['ctoken'][0]
         video_id = ctoken_metadata(ctoken)['video_id']
@@ -200,7 +201,7 @@ def parse_comments_polymer(content, replies=False):
             comments_raw = content[1]['response']['continuationContents']['commentRepliesContinuation']['contents']
             replies = True
 
-        ctoken = default_multi_get(content, 1, 'response', 'continuationContents', 'commentSectionContinuation', 'continuations', 0, 'nextContinuationData', 'continuation', default='')
+        ctoken = util.default_multi_get(content, 1, 'response', 'continuationContents', 'commentSectionContinuation', 'continuations', 0, 'nextContinuationData', 'continuation', default='')
         
         comments = []
         for comment_raw in comments_raw:
@@ -219,8 +220,8 @@ def parse_comments_polymer(content, replies=False):
                 if 'replies' in comment_raw:
                     #reply_ctoken = comment_raw['replies']['commentRepliesRenderer']['continuations'][0]['nextContinuationData']['continuation']
                     #comment_id, video_id = get_ids(reply_ctoken)
-                    replies_url = URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id
-                    view_replies_text = common.get_plain_text(comment_raw['replies']['commentRepliesRenderer']['moreText'])
+                    replies_url = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id
+                    view_replies_text = yt_data_extract.get_plain_text(comment_raw['replies']['commentRepliesRenderer']['moreText'])
                     match = reply_count_regex.search(view_replies_text)
                     if match is None:
                         view_replies_text = '1 reply'
@@ -228,24 +229,31 @@ def parse_comments_polymer(content, replies=False):
                         view_replies_text = match.group(1) + " replies"
                 elif not replies:
                     view_replies_text = "Reply"
-                    replies_url = URL_ORIGIN + '/post_comment?parent_id=' + parent_id + "&video_id=" + video_id
+                    replies_url = util.URL_ORIGIN + '/post_comment?parent_id=' + parent_id + "&video_id=" + video_id
                 comment_raw = comment_raw['comment']
             
             comment_raw = comment_raw['commentRenderer']
             comment = {
-            'author': common.get_plain_text(comment_raw['authorText']),
-            'author_url': comment_raw['authorEndpoint']['commandMetadata']['webCommandMetadata']['url'],
-            'author_channel_id': comment_raw['authorEndpoint']['browseEndpoint']['browseId'],
-            'author_id': comment_raw['authorId'],
+            'author_id': comment_raw.get('authorId', ''),
             'author_avatar': comment_raw['authorThumbnail']['thumbnails'][0]['url'],
             'likes': comment_raw['likeCount'],
-            'published': common.get_plain_text(comment_raw['publishedTimeText']),
+            'published': yt_data_extract.get_plain_text(comment_raw['publishedTimeText']),
             'text': comment_raw['contentText'].get('runs', ''),
             'view_replies_text': view_replies_text,
             'replies_url': replies_url,
             'video_id': video_id,
             'comment_id': comment_raw['commentId'],
             }
+
+            if 'authorText' in comment_raw:     # deleted channels have no name or channel link
+                comment['author'] = yt_data_extract.get_plain_text(comment_raw['authorText'])
+                comment['author_url'] = comment_raw['authorEndpoint']['commandMetadata']['webCommandMetadata']['url']
+                comment['author_channel_id'] = comment_raw['authorEndpoint']['browseEndpoint']['browseId']
+            else:
+                comment['author'] = ''
+                comment['author_url'] = ''
+                comment['author_channel_id'] = ''
+
             comments.append(comment)
     except Exception as e:
         print('Error parsing comments: ' + str(e))
@@ -264,13 +272,13 @@ def get_comments_html(comments):
             replies = reply_link_template.substitute(url=comment['replies_url'], view_replies_text=html.escape(comment['view_replies_text']))
         if settings.enable_comment_avatars:
             avatar = comment_avatar_template.substitute(
-                author_url = URL_ORIGIN + comment['author_url'],
+                author_url = util.URL_ORIGIN + comment['author_url'],
                 author_avatar = '/' + comment['author_avatar'],
             )
         else:
             avatar = ''
         if comment['author_channel_id'] in accounts.accounts:
-            delete_url = (URL_ORIGIN + '/delete_comment?video_id='
+            delete_url = (util.URL_ORIGIN + '/delete_comment?video_id='
                 + comment['video_id']
                 + '&channel_id='+ comment['author_channel_id']
                 + '&author_id=' + comment['author_id']
@@ -280,14 +288,14 @@ def get_comments_html(comments):
         else:
             action_buttons = ''
 
-        permalink = URL_ORIGIN + '/watch?v=' + comment['video_id'] + '&lc=' + comment['comment_id']
+        permalink = util.URL_ORIGIN + '/watch?v=' + comment['video_id'] + '&lc=' + comment['comment_id']
         html_result += comment_template.substitute(
             author=comment['author'],
-            author_url = URL_ORIGIN + comment['author_url'],
+            author_url = util.URL_ORIGIN + comment['author_url'],
             avatar = avatar,
             likes = str(comment['likes']) + ' likes' if str(comment['likes']) != '0' else '',
             published = comment['published'],
-            text = format_text_runs(comment['text']),
+            text = yt_data_extract.format_text_runs(comment['text']),
             datetime = '',  #TODO
             replies = replies,
             action_buttons = action_buttons,
@@ -297,10 +305,10 @@ def get_comments_html(comments):
     
 def video_comments(video_id, sort=0, offset=0, lc='', secret_key=''):
     if settings.enable_comments:
-        post_comment_url = common.URL_ORIGIN + "/post_comment?video_id=" + video_id
+        post_comment_url = util.URL_ORIGIN + "/post_comment?video_id=" + video_id
         post_comment_link = '''<a class="sort-button" href="''' + post_comment_url + '''">Post comment</a>'''
 
-        other_sort_url = common.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(video_id, sort=1 - sort, lc=lc)
+        other_sort_url = util.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(video_id, sort=1 - sort, lc=lc)
         other_sort_name = 'newest' if sort == 0 else 'top'
         other_sort_link = '''<a class="sort-button" href="''' + other_sort_url + '''">Sort by ''' + other_sort_name + '''</a>'''
 
@@ -314,7 +322,7 @@ def video_comments(video_id, sort=0, offset=0, lc='', secret_key=''):
         if ctoken == '':
             more_comments_button = ''
         else:
-            more_comments_button = more_comments_template.substitute(url = common.URL_ORIGIN + '/comments?ctoken=' + ctoken)
+            more_comments_button = more_comments_template.substitute(url = util.URL_ORIGIN + '/comments?ctoken=' + ctoken)
 
         result = '''<section class="comments-area">\n'''
         result += comment_links + '\n'
@@ -350,7 +358,7 @@ comment_box_template = Template('''
         <select id="account-selection" name="channel_id">
 $options
         </select>
-        <a href="''' + common.URL_ORIGIN + '''/login" target="_blank">Add account</a>
+        <a href="''' + util.URL_ORIGIN + '''/login" target="_blank">Add account</a>
     </div>
     <textarea name="comment_text"></textarea>
     $video_id_input
@@ -359,7 +367,7 @@ $options
 def get_comments_page(env, start_response):
     start_response('200 OK',  [('Content-type','text/html'),] )
     parameters = env['parameters']
-    ctoken = default_multi_get(parameters, 'ctoken', 0, default='')
+    ctoken = util.default_multi_get(parameters, 'ctoken', 0, default='')
     replies = False
     if not ctoken:
         video_id = parameters['video_id'][0]
@@ -384,17 +392,17 @@ def get_comments_page(env, start_response):
             page_number = page_number,
             sort = 'top' if metadata['sort'] == 0 else 'newest',
             title = html.escape(comment_info['video_title']),
-            url = common.URL_ORIGIN + '/watch?v=' + metadata['video_id'],
+            url = util.URL_ORIGIN + '/watch?v=' + metadata['video_id'],
             thumbnail = '/i.ytimg.com/vi/'+ metadata['video_id'] + '/mqdefault.jpg',
         )
         comment_box = comment_box_template.substitute(
-            form_action= common.URL_ORIGIN + '/post_comment',
+            form_action= util.URL_ORIGIN + '/post_comment',
             video_id_input='''<input type="hidden" name="video_id" value="''' + metadata['video_id'] + '''">''',
             post_text='Post comment',
             options=comment_box_account_options(),
         )
 
-        other_sort_url = common.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(metadata['video_id'], sort=1 - metadata['sort'])
+        other_sort_url = util.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(metadata['video_id'], sort=1 - metadata['sort'])
         other_sort_name = 'newest' if metadata['sort'] == 0 else 'top'
         other_sort_link = '''<a class="sort-button" href="''' + other_sort_url + '''">Sort by ''' + other_sort_name + '''</a>'''
 
@@ -408,7 +416,7 @@ def get_comments_page(env, start_response):
     if ctoken == '':
         more_comments_button = ''
     else:
-        more_comments_button = more_comments_template.substitute(url = URL_ORIGIN + '/comments?ctoken=' + ctoken)
+        more_comments_button = more_comments_template.substitute(url = util.URL_ORIGIN + '/comments?ctoken=' + ctoken)
     comments_area = '<section class="comments-area">\n'
     comments_area += video_metadata + comment_box + comment_links + '\n'
     comments_area += '<div class="comments">\n'
@@ -417,7 +425,7 @@ def get_comments_page(env, start_response):
     comments_area += more_comments_button + '\n'
     comments_area += '</section>\n'
     return yt_comments_template.substitute(
-        header = common.get_header(),
+        header = html_common.get_header(),
         comments_area = comments_area,
         page_title = page_title,
     ).encode('utf-8')
diff --git a/youtube/common.py b/youtube/html_common.py
index cb963ce..8e65a1f 100644
--- a/youtube/common.py
+++ b/youtube/html_common.py
@@ -1,46 +1,8 @@
 from youtube.template import Template
-from youtube import local_playlist
-import settings
-import html
+from youtube import local_playlist, yt_data_extract, util
+
 import json
-import re
-import urllib.parse
-import gzip
-import brotli
-import time
-import socks, sockshandler
-
-URL_ORIGIN = "/https://www.youtube.com"
-
-
-# videos (all of type str):
-
-# id
-# title
-# url
-# author
-# author_url
-# thumbnail
-# description
-# published
-# duration
-# likes
-# dislikes
-# views
-# playlist_index
-
-# playlists:
-
-# id
-# title
-# url
-# author
-# author_url
-# thumbnail
-# description
-# updated
-# size
-# first_video_id
+import html
 
 
 with open('yt_basic_template.html', 'r', encoding='utf-8') as file:
@@ -139,205 +101,8 @@ medium_channel_item_template = Template('''
 ''')
 
 
-class HTTPAsymmetricCookieProcessor(urllib.request.BaseHandler):
-    '''Separate cookiejars for receiving and sending'''
-    def __init__(self, cookiejar_send=None, cookiejar_receive=None):
-        import http.cookiejar
-        self.cookiejar_send = cookiejar_send
-        self.cookiejar_receive = cookiejar_receive
-
-    def http_request(self, request):
-        if self.cookiejar_send is not None:
-            self.cookiejar_send.add_cookie_header(request)
-        return request
-
-    def http_response(self, request, response):
-        if self.cookiejar_receive is not None:
-            self.cookiejar_receive.extract_cookies(response, request)
-        return response
-
-    https_request = http_request
-    https_response = http_response
-
-
-def decode_content(content, encoding_header):
-    encodings = encoding_header.replace(' ', '').split(',')
-    for encoding in reversed(encodings):
-        if encoding == 'identity':
-            continue
-        if encoding == 'br':
-            content = brotli.decompress(content)
-        elif encoding == 'gzip':
-            content = gzip.decompress(content)
-    return content
-
-def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True):
-    '''
-    When cookiejar_send is set to a CookieJar object,
-     those cookies will be sent in the request (but cookies in response will not be merged into it)
-    When cookiejar_receive is set to a CookieJar object,
-     cookies received in the response will be merged into the object (nothing will be sent from it)
-    When both are set to the same object, cookies will be sent from the object,
-     and response cookies will be merged into it.
-    '''
-    headers = dict(headers)     # Note: Calling dict() on a dict will make a copy
-    headers['Accept-Encoding'] = 'gzip, br'
-
-    # prevent python version being leaked by urllib if User-Agent isn't provided
-    #  (urllib will use ex. Python-urllib/3.6 otherwise)
-    if 'User-Agent' not in headers and 'user-agent' not in headers and 'User-agent' not in headers:
-        headers['User-Agent'] = 'Python-urllib'
-
-    if data is not None:
-        if isinstance(data, str):
-            data = data.encode('ascii')
-        elif not isinstance(data, bytes):
-            data = urllib.parse.urlencode(data).encode('ascii')
-
-    start_time = time.time()
-
-
-    req = urllib.request.Request(url, data=data, headers=headers)
-
-    cookie_processor = HTTPAsymmetricCookieProcessor(cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive)
-
-    if use_tor and settings.route_tor:
-        opener = urllib.request.build_opener(sockshandler.SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150), cookie_processor)
-    else:
-        opener = urllib.request.build_opener(cookie_processor)
-
-    response = opener.open(req, timeout=timeout)
-    response_time = time.time()
-
-
-    content = response.read()
-    read_finish = time.time()
-    if report_text:
-        print(report_text, '    Latency:', round(response_time - start_time,3), '    Read time:', round(read_finish - response_time,3))
-    content = decode_content(content, response.getheader('Content-Encoding', default='identity'))
-    return content
-
-mobile_user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1'
-mobile_ua = (('User-Agent', mobile_user_agent),)
-desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
-desktop_ua = (('User-Agent', desktop_user_agent),)
 
-def dict_add(*dicts):
-    for dictionary in dicts[1:]:
-        dicts[0].update(dictionary)
-    return dicts[0]
 
-def video_id(url):
-    url_parts = urllib.parse.urlparse(url)
-    return urllib.parse.parse_qs(url_parts.query)['v'][0]
-
-def uppercase_escape(s):
-     return re.sub(
-         r'\\U([0-9a-fA-F]{8})',
-         lambda m: chr(int(m.group(1), base=16)), s)
-
-def default_multi_get(object, *keys, default):
-    ''' Like dict.get(), but for nested dictionaries/sequences, supporting keys or indices. Last argument is the default value to use in case of any IndexErrors or KeyErrors '''
-    try:
-        for key in keys:
-            object = object[key]
-        return object
-    except (IndexError, KeyError):
-        return default
-
-def get_plain_text(node):
-    try:
-        return html.escape(node['simpleText'])
-    except KeyError:
-        return unformmated_text_runs(node['runs'])
-        
-def unformmated_text_runs(runs):
-    result = ''
-    for text_run in runs:
-        result += html.escape(text_run["text"])
-    return result
-
-def format_text_runs(runs):
-    if isinstance(runs, str):
-        return runs
-    result = ''
-    for text_run in runs:
-        if text_run.get("bold", False):
-            result += "<b>" + html.escape(text_run["text"]) + "</b>"
-        elif text_run.get('italics', False):
-            result += "<i>" + html.escape(text_run["text"]) + "</i>"
-        else:
-            result += html.escape(text_run["text"])
-    return result
-
-# default, sddefault, mqdefault, hqdefault, hq720
-def get_thumbnail_url(video_id):
-    return "/i.ytimg.com/vi/" + video_id + "/mqdefault.jpg"
-    
-def seconds_to_timestamp(seconds):
-    seconds = int(seconds)
-    hours, seconds = divmod(seconds,3600)
-    minutes, seconds = divmod(seconds,60)
-    if hours != 0:
-        timestamp = str(hours) + ":"
-        timestamp += str(minutes).zfill(2)  # zfill pads with zeros
-    else:
-        timestamp = str(minutes)
-
-    timestamp += ":" + str(seconds).zfill(2)
-    return timestamp
-
-
-# -----
-# HTML
-# -----
-
-def small_video_item_html(item):
-    video_info = json.dumps({key: item[key] for key in ('id', 'title', 'author', 'duration')})
-    return small_video_item_template.substitute(
-        title       = html.escape(item["title"]),
-        views       = item["views"],
-        author      = html.escape(item["author"]),
-        duration    = item["duration"],
-        url         = URL_ORIGIN + "/watch?v=" + item["id"],
-        thumbnail   = get_thumbnail_url(item['id']),
-        video_info  = html.escape(video_info),
-    )
-
-def small_playlist_item_html(item):
-    return small_playlist_item_template.substitute(
-        title=html.escape(item["title"]),
-        size = item['size'],
-        author="",
-        url = URL_ORIGIN + "/playlist?list=" + item["id"],
-        thumbnail= get_thumbnail_url(item['first_video_id']),
-    )
-
-def medium_playlist_item_html(item):
-    return medium_playlist_item_template.substitute(
-        title=html.escape(item["title"]),
-        size = item['size'],
-        author=item['author'],
-        author_url= URL_ORIGIN + item['author_url'],
-        url = URL_ORIGIN + "/playlist?list=" + item["id"],
-        thumbnail= item['thumbnail'],
-    )
-
-def medium_video_item_html(medium_video_info):
-    info = medium_video_info
-       
-    return medium_video_item_template.substitute(
-            title=html.escape(info["title"]),
-            views=info["views"],
-            published = info["published"],
-            description = format_text_runs(info["description"]),
-            author=html.escape(info["author"]),
-            author_url=info["author_url"],
-            duration=info["duration"],
-            url = URL_ORIGIN + "/watch?v=" + info["id"],
-            thumbnail=info['thumbnail'],
-            datetime='', # TODO
-        )
 
 
 header_template = Template('''
@@ -440,158 +205,28 @@ def get_header(search_box_value=""):
 
 
 
-def get_url(node):
-    try:
-        return node['runs'][0]['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
-    except KeyError:
-        return node['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
 
 
-def get_text(node):
-    try:
-        return node['simpleText']
-    except KeyError:
-            pass
-    try:
-        return node['runs'][0]['text']
-    except IndexError: # empty text runs
-        return ''
 
-def get_formatted_text(node):
-    try:
-        return node['runs']
-    except KeyError:
-        return node['simpleText']
 
-def get_badges(node):
-    badges = []
-    for badge_node in node:
-        badge = badge_node['metadataBadgeRenderer']['label']
-        if badge.lower() != 'new':
-            badges.append(badge)
-    return badges
 
-def get_thumbnail(node):
-    try:
-        return node['thumbnails'][0]['url']     # polymer format
-    except KeyError:
-        return node['url']     # ajax format
-
-dispatch = {
-
-# polymer format    
-    'title':                ('title',       get_text),
-    'publishedTimeText':    ('published',   get_text),
-    'videoId':              ('id',          lambda node: node),
-    'descriptionSnippet':   ('description', get_formatted_text),
-    'lengthText':           ('duration',    get_text),
-    'thumbnail':            ('thumbnail',   get_thumbnail),
-    'thumbnails':           ('thumbnail',   lambda node: node[0]['thumbnails'][0]['url']),
-
-    'viewCountText':        ('views',       get_text),
-    'numVideosText':        ('size',        lambda node: get_text(node).split(' ')[0]),     # the format is "324 videos"
-    'videoCountText':       ('size',        get_text),
-    'playlistId':           ('id',          lambda node: node),
-    'descriptionText':      ('description', get_formatted_text),
-
-    'subscriberCountText':  ('subscriber_count',    get_text),
-    'channelId':            ('id',          lambda node: node),
-    'badges':               ('badges',      get_badges),
-
-# ajax format
-    'view_count_text':  ('views',       get_text),
-    'num_videos_text':  ('size',        lambda node: get_text(node).split(' ')[0]),
-    'owner_text':       ('author',      get_text),
-    'owner_endpoint':   ('author_url',  lambda node: node['url']),
-    'description':      ('description', get_formatted_text),
-    'index':            ('playlist_index', get_text),
-    'short_byline':     ('author',      get_text),
-    'length':           ('duration',    get_text),
-    'video_id':         ('id',          lambda node: node),
 
-}
 
-def renderer_info(renderer):
-    try:
-        info = {}
-        if 'viewCountText' in renderer:     # prefer this one as it contains all the digits
-            info['views'] = get_text(renderer['viewCountText'])
-        elif 'shortViewCountText' in renderer:
-            info['views'] = get_text(renderer['shortViewCountText'])
-
-        if 'ownerText' in renderer:
-            info['author'] = renderer['ownerText']['runs'][0]['text']
-            info['author_url'] = renderer['ownerText']['runs'][0]['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
-        try:
-            overlays = renderer['thumbnailOverlays']
-        except KeyError:
-            pass
-        else:
-            for overlay in overlays:
-                if 'thumbnailOverlayTimeStatusRenderer' in overlay:
-                    info['duration'] = get_text(overlay['thumbnailOverlayTimeStatusRenderer']['text'])
-                # show renderers don't have videoCountText
-                elif 'thumbnailOverlayBottomPanelRenderer' in overlay:
-                    info['size'] = get_text(overlay['thumbnailOverlayBottomPanelRenderer']['text'])
-
-        # show renderers don't have playlistId, have to dig into the url to get it
-        try:
-            info['id'] = renderer['navigationEndpoint']['watchEndpoint']['playlistId']
-        except KeyError:
-            pass
-        for key, node in renderer.items():
-            if key in ('longBylineText', 'shortBylineText'):
-                info['author'] = get_text(node)
-                try:
-                    info['author_url'] = get_url(node)
-                except KeyError:
-                    pass
-
-            # show renderers don't have thumbnail key at top level, dig into thumbnailRenderer
-            elif key == 'thumbnailRenderer' and 'showCustomThumbnailRenderer' in node:
-                info['thumbnail'] = node['showCustomThumbnailRenderer']['thumbnail']['thumbnails'][0]['url']
-            else:
-                try:
-                    simple_key, function = dispatch[key]
-                except KeyError:
-                    continue
-                info[simple_key] = function(node)
-        return info
-    except KeyError:
-        print(renderer)
-        raise
-    
-def ajax_info(item_json):
-    try:
-        info = {}          
-        for key, node in item_json.items():
-            try:
-                simple_key, function = dispatch[key]
-            except KeyError:
-                continue
-            info[simple_key] = function(node)
-        return info
-    except KeyError:
-        print(item_json)
-        raise
-    
+
 def badges_html(badges):
     return ' | '.join(map(html.escape, badges))
 
 
-
-
-
 html_transform_dispatch = {
     'title':        html.escape,
     'published':    html.escape,
     'id':           html.escape,
-    'description':  format_text_runs,
+    'description':  yt_data_extract.format_text_runs,
     'duration':     html.escape,
     'thumbnail':    lambda url: html.escape('/' + url.lstrip('/')),
     'size':         html.escape,
     'author':       html.escape,
-    'author_url':   lambda url: html.escape(URL_ORIGIN + url),
+    'author_url':   lambda url: html.escape(util.URL_ORIGIN + url),
     'views':        html.escape,
     'subscriber_count': html.escape,
     'badges':       badges_html,
@@ -645,7 +280,7 @@ def video_item_html(item, template, html_exclude=set()):
     html_ready = get_html_ready(item)
 
     html_ready['video_info'] = html.escape(json.dumps(video_info) )
-    html_ready['url'] = URL_ORIGIN + "/watch?v=" + html_ready['id']
+    html_ready['url'] = util.URL_ORIGIN + "/watch?v=" + html_ready['id']
     html_ready['datetime'] = '' #TODO
     
     for key in html_exclude:
@@ -658,7 +293,7 @@ def video_item_html(item, template, html_exclude=set()):
 def playlist_item_html(item, template, html_exclude=set()):
     html_ready = get_html_ready(item)
 
-    html_ready['url'] = URL_ORIGIN + "/playlist?list=" + html_ready['id']
+    html_ready['url'] = util.URL_ORIGIN + "/playlist?list=" + html_ready['id']
     html_ready['datetime'] = '' #TODO
 
     for key in html_exclude:
@@ -672,10 +307,6 @@ def playlist_item_html(item, template, html_exclude=set()):
 
 
 
-def update_query_string(query_string, items):
-    parameters = urllib.parse.parse_qs(query_string)
-    parameters.update(items)
-    return urllib.parse.urlencode(parameters, doseq=True)
 
 page_button_template = Template('''<a class="page-button" href="$href">$page</a>''')
 current_page_button_template = Template('''<div class="page-button">$page</div>''')
@@ -694,7 +325,7 @@ def page_buttons_html(current_page, estimated_pages, url, current_query_string):
             template = current_page_button_template
         else:
             template = page_button_template
-        result += template.substitute(page=page, href = url + "?" + update_query_string(current_query_string, {'page': [str(page)]}) )
+        result += template.substitute(page=page, href = url + "?" + util.update_query_string(current_query_string, {'page': [str(page)]}) )
     return result
 
 
@@ -723,15 +354,15 @@ def renderer_html(renderer, additional_info={}, current_query_string=''):
         return renderer_html(renderer['contents'][0], additional_info, current_query_string)
 
     if type == 'channelRenderer':
-        info = renderer_info(renderer)
+        info = yt_data_extract.renderer_info(renderer)
         html_ready = get_html_ready(info)
-        html_ready['url'] = URL_ORIGIN + "/channel/" + html_ready['id']
+        html_ready['url'] = util.URL_ORIGIN + "/channel/" + html_ready['id']
         return medium_channel_item_template.substitute(html_ready)
     
     if type in ('movieRenderer', 'clarificationRenderer'):
         return ''
 
-    info = renderer_info(renderer)
+    info = yt_data_extract.renderer_info(renderer)
     info.update(additional_info)
     html_exclude = set(additional_info.keys())
     if type == 'compactVideoRenderer':
@@ -745,4 +376,4 @@ def renderer_html(renderer, additional_info={}, current_query_string=''):
 
     #print(renderer)
     #raise NotImplementedError('Unknown renderer type: ' + type)
-    return ''
+    return ''
\ No newline at end of file
diff --git a/youtube/local_playlist.py b/youtube/local_playlist.py
index 0375040..e354013 100644
--- a/youtube/local_playlist.py
+++ b/youtube/local_playlist.py
@@ -1,11 +1,12 @@
+from youtube.template import Template
+from youtube import util, html_common
+import settings
+
 import os
 import json
-from youtube.template import Template
-from youtube import common
 import html
 import gevent
 import urllib
-import settings
 
 playlists_directory = os.path.join(settings.data_dir, "playlists")
 thumbnails_directory = os.path.join(settings.data_dir, "playlist_thumbnails")
@@ -38,7 +39,7 @@ def download_thumbnail(playlist_name, video_id):
     url = "https://i.ytimg.com/vi/" + video_id + "/mqdefault.jpg"
     save_location = os.path.join(thumbnails_directory, playlist_name, video_id + ".jpg")
     try:
-        thumbnail = common.fetch_url(url, report_text="Saved local playlist thumbnail: " + video_id)
+        thumbnail = util.fetch_url(url, report_text="Saved local playlist thumbnail: " + video_id)
     except urllib.error.HTTPError as e:
         print("Failed to download thumbnail for " + video_id + ": " + str(e))
         return
@@ -78,15 +79,15 @@ def get_local_playlist_page(name):
             if info['id'] + ".jpg" in thumbnails:
                 info['thumbnail'] = "/youtube.com/data/playlist_thumbnails/" + name + "/" + info['id'] + ".jpg"
             else:
-                info['thumbnail'] = common.get_thumbnail_url(info['id'])
+                info['thumbnail'] = util.get_thumbnail_url(info['id'])
                 missing_thumbnails.append(info['id'])
-            videos_html += common.video_item_html(info, common.small_video_item_template)
+            videos_html += html_common.video_item_html(info, html_common.small_video_item_template)
         except json.decoder.JSONDecodeError:
             pass
     gevent.spawn(download_thumbnails, name, missing_thumbnails)
     return local_playlist_template.substitute(
         page_title = name + ' - Local playlist',
-        header = common.get_header(),
+        header = html_common.get_header(),
         videos = videos_html,
         title = name,
         page_buttons = ''
@@ -127,11 +128,11 @@ def get_playlists_list_page():
     page = '''<ul>\n'''
     list_item_template = Template('''    <li><a href="$url">$name</a></li>\n''')
     for name in get_playlist_names():
-        page += list_item_template.substitute(url = html.escape(common.URL_ORIGIN + '/playlists/' + name), name = html.escape(name))
+        page += list_item_template.substitute(url = html.escape(util.URL_ORIGIN + '/playlists/' + name), name = html.escape(name))
     page += '''</ul>\n'''
-    return common.yt_basic_template.substitute(
+    return html_common.yt_basic_template.substitute(
         page_title = "Local playlists",
-        header = common.get_header(),
+        header = html_common.get_header(),
         style = '',
         page = page,
     )
@@ -151,7 +152,7 @@ def path_edit_playlist(env, start_response):
     if parameters['action'][0] == 'remove':
         playlist_name = env['path_parts'][1]
         remove_from_playlist(playlist_name, parameters['video_info_list'])
-        start_response('303 See Other', [('Location', common.URL_ORIGIN + env['PATH_INFO']),] )
+        start_response('303 See Other', [('Location', util.URL_ORIGIN + env['PATH_INFO']),] )
         return b''
 
     else:
diff --git a/youtube/playlist.py b/youtube/playlist.py
index cc0da33..fbe6448 100644
--- a/youtube/playlist.py
+++ b/youtube/playlist.py
@@ -1,14 +1,14 @@
+from youtube import util, yt_data_extract, html_common, template, proto
+
 import base64
-import youtube.common as common
 import urllib
 import json
-from string import Template
-import youtube.proto as proto
+import string
 import gevent
 import math
 
 with open("yt_playlist_template.html", "r") as file:
-    yt_playlist_template = Template(file.read())
+    yt_playlist_template = template.Template(file.read())
 
 
 
@@ -48,10 +48,10 @@ headers_1 = (
 
 def playlist_first_page(playlist_id, report_text = "Retrieved playlist"):
     url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1'
-    content = common.fetch_url(url, common.mobile_ua + headers_1, report_text=report_text)
+    content = util.fetch_url(url, util.mobile_ua + headers_1, report_text=report_text)
     '''with open('debug/playlist_debug', 'wb') as f:
         f.write(content)'''
-    content = json.loads(common.uppercase_escape(content.decode('utf-8')))
+    content = json.loads(util.uppercase_escape(content.decode('utf-8')))
 
     return content
     
@@ -68,15 +68,15 @@ def get_videos(playlist_id, page):
         'X-YouTube-Client-Version': '2.20180508',
     }
 
-    content = common.fetch_url(url, headers, report_text="Retrieved playlist")
+    content = util.fetch_url(url, headers, report_text="Retrieved playlist")
     '''with open('debug/playlist_debug', 'wb') as f:
         f.write(content)'''
 
-    info = json.loads(common.uppercase_escape(content.decode('utf-8')))
+    info = json.loads(util.uppercase_escape(content.decode('utf-8')))
     return info
 
 
-playlist_stat_template = Template('''
+playlist_stat_template = string.Template('''
 <div>$stat</div>''')
 def get_playlist_page(env, start_response):
     start_response('200 OK', [('Content-type','text/html'),])
@@ -100,22 +100,22 @@ def get_playlist_page(env, start_response):
         video_list = this_page_json['response']['continuationContents']['playlistVideoListContinuation']['contents']
     videos_html = ''
     for video_json in video_list:
-        info = common.renderer_info(video_json['playlistVideoRenderer'])
-        videos_html += common.video_item_html(info, common.small_video_item_template)
+        info = yt_data_extract.renderer_info(video_json['playlistVideoRenderer'])
+        videos_html += html_common.video_item_html(info, html_common.small_video_item_template)
 
 
-    metadata = common.renderer_info(first_page_json['response']['header']['playlistHeaderRenderer'])
+    metadata = yt_data_extract.renderer_info(first_page_json['response']['header']['playlistHeaderRenderer'])
     video_count = int(metadata['size'].replace(',', ''))
-    page_buttons = common.page_buttons_html(int(page), math.ceil(video_count/20), common.URL_ORIGIN + "/playlist", env['QUERY_STRING'])
+    page_buttons = html_common.page_buttons_html(int(page), math.ceil(video_count/20), util.URL_ORIGIN + "/playlist", env['QUERY_STRING'])
 
-    html_ready = common.get_html_ready(metadata)
+    html_ready = html_common.get_html_ready(metadata)
     html_ready['page_title'] = html_ready['title'] + ' - Page ' + str(page)
 
     stats = ''
     stats += playlist_stat_template.substitute(stat=html_ready['size'] + ' videos')
     stats += playlist_stat_template.substitute(stat=html_ready['views'])
     return yt_playlist_template.substitute(
-        header          = common.get_header(),
+        header          = html_common.get_header(),
         videos          = videos_html,
         page_buttons    = page_buttons,
         stats = stats,
diff --git a/youtube/post_comment.py b/youtube/post_comment.py
index 92c45e1..876a1c0 100644
--- a/youtube/post_comment.py
+++ b/youtube/post_comment.py
@@ -1,11 +1,11 @@
 # Contains functions having to do with posting/editing/deleting comments
+from youtube import util, html_common, proto, comments, accounts
+import settings
 
 import urllib
 import json
-from youtube import common, proto, comments, accounts
 import re
 import traceback
-import settings
 import os
 
 def _post_comment(text, video_id, session_token, cookiejar):
@@ -31,7 +31,7 @@ def _post_comment(text, video_id, session_token, cookiejar):
     data = urllib.parse.urlencode(data_dict).encode()
 
 
-    content = common.fetch_url("https://m.youtube.com/service_ajax?name=createCommentEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
+    content = util.fetch_url("https://m.youtube.com/service_ajax?name=createCommentEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
 
     code = json.loads(content)['code']
     print("Comment posting code: " + code)
@@ -62,7 +62,7 @@ def _post_comment_reply(text, video_id, parent_comment_id, session_token, cookie
     }
     data = urllib.parse.urlencode(data_dict).encode()
 
-    content = common.fetch_url("https://m.youtube.com/service_ajax?name=createCommentReplyEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
+    content = util.fetch_url("https://m.youtube.com/service_ajax?name=createCommentReplyEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
 
     code = json.loads(content)['code']
     print("Comment posting code: " + code)
@@ -90,7 +90,7 @@ def _delete_comment(video_id, comment_id, author_id, session_token, cookiejar):
     }
     data = urllib.parse.urlencode(data_dict).encode()
 
-    content = common.fetch_url("https://m.youtube.com/service_ajax?name=performCommentActionEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
+    content = util.fetch_url("https://m.youtube.com/service_ajax?name=performCommentActionEndpoint", headers=headers, data=data, cookiejar_send=cookiejar)
     code = json.loads(content)['code']
     print("Comment deletion code: " + code)
     return code
@@ -101,8 +101,8 @@ def get_session_token(video_id, cookiejar):
     # youtube-dl uses disable_polymer=1 which uses a different request format which has an obfuscated javascript algorithm to generate a parameter called "bgr"
     # Tokens retrieved from disable_polymer pages only work with that format. Tokens retrieved on mobile only work using mobile requests
     # Additionally, tokens retrieved without sending the same cookie won't work. So this is necessary even if the bgr and stuff was reverse engineered.
-    headers = {'User-Agent': common.mobile_user_agent}
-    mobile_page = common.fetch_url('https://m.youtube.com/watch?v=' + video_id, headers, report_text="Retrieved session token for comment", cookiejar_send=cookiejar, cookiejar_receive=cookiejar).decode()
+    headers = {'User-Agent': util.mobile_user_agent}
+    mobile_page = util.fetch_url('https://m.youtube.com/watch?v=' + video_id, headers, report_text="Retrieved session token for comment", cookiejar_send=cookiejar, cookiejar_receive=cookiejar).decode()
     match = xsrf_token_regex.search(mobile_page)
     if match:
         return match.group(1).replace("%3D", "=")
@@ -118,9 +118,9 @@ def delete_comment(env, start_response):
     code = _delete_comment(video_id, parameters['comment_id'][0], parameters['author_id'][0], token, cookiejar)
 
     if code == "SUCCESS":
-        start_response('303 See Other',  [('Location', common.URL_ORIGIN + '/comment_delete_success'),] )
+        start_response('303 See Other',  [('Location', util.URL_ORIGIN + '/comment_delete_success'),] )
     else:
-        start_response('303 See Other',  [('Location', common.URL_ORIGIN + '/comment_delete_fail'),] )
+        start_response('303 See Other',  [('Location', util.URL_ORIGIN + '/comment_delete_fail'),] )
 
 def post_comment(env, start_response):
     parameters = env['parameters']
@@ -131,11 +131,11 @@ def post_comment(env, start_response):
 
     if 'parent_id' in parameters:
         code = _post_comment_reply(parameters['comment_text'][0], parameters['video_id'][0], parameters['parent_id'][0], token, cookiejar)
-        start_response('303 See Other',  (('Location', common.URL_ORIGIN + '/comments?' + env['QUERY_STRING']),) )
+        start_response('303 See Other',  (('Location', util.URL_ORIGIN + '/comments?' + env['QUERY_STRING']),) )
 
     else:
         code = _post_comment(parameters['comment_text'][0], parameters['video_id'][0], token, cookiejar)
-        start_response('303 See Other',  (('Location', common.URL_ORIGIN + '/comments?ctoken=' + comments.make_comment_ctoken(video_id, sort=1)),) )
+        start_response('303 See Other',  (('Location', util.URL_ORIGIN + '/comments?ctoken=' + comments.make_comment_ctoken(video_id, sort=1)),) )
 
     return b''
 
@@ -163,10 +163,10 @@ def get_delete_comment_page(env, start_response):
     page += '''
         <input type="submit" value="Yes, delete it">
     </form>'''
-    return common.yt_basic_template.substitute(
+    return html_common.yt_basic_template.substitute(
         page_title = "Delete comment?",
         style = style,
-        header = common.get_header(),
+        header = html_common.get_header(),
         page = page,
     ).encode('utf-8')
 
@@ -174,7 +174,7 @@ def get_post_comment_page(env, start_response):
     start_response('200 OK', [('Content-type','text/html'),])
     parameters = env['parameters']
     video_id = parameters['video_id'][0]
-    parent_id = common.default_multi_get(parameters, 'parent_id', 0, default='')
+    parent_id = util.default_multi_get(parameters, 'parent_id', 0, default='')
     
     style = ''' main{
     display: grid;
@@ -194,23 +194,23 @@ textarea{
 }'''
     if parent_id:   # comment reply
         comment_box = comments.comment_box_template.substitute(
-            form_action = common.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id,
+            form_action = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id,
             video_id_input = '',
             post_text = "Post reply",
             options=comments.comment_box_account_options(),
         )
     else:
         comment_box = comments.comment_box_template.substitute(
-            form_action = common.URL_ORIGIN + '/post_comment',
+            form_action = util.URL_ORIGIN + '/post_comment',
             video_id_input = '''<input type="hidden" name="video_id" value="''' + video_id + '''">''',
             post_text = "Post comment",
             options=comments.comment_box_account_options(),
         )
         
     page = '''<div class="left">\n''' + comment_box + '''</div>\n'''
-    return common.yt_basic_template.substitute(
+    return html_common.yt_basic_template.substitute(
         page_title = "Post comment reply" if parent_id else "Post a comment",
         style = style,
-        header = common.get_header(),
+        header = html_common.get_header(),
         page = page,
     ).encode('utf-8')
diff --git a/youtube/proto.py b/youtube/proto.py
index 004375a..d966455 100644
--- a/youtube/proto.py
+++ b/youtube/proto.py
@@ -60,7 +60,7 @@ def unpadded_b64encode(data):
 
 def as_bytes(value):
     if isinstance(value, str):
-        return value.encode('ascii')
+        return value.encode('utf-8')
     return value
 
 
diff --git a/youtube/search.py b/youtube/search.py
index db65eaa..0cef0f3 100644
--- a/youtube/search.py
+++ b/youtube/search.py
@@ -1,11 +1,12 @@
+from youtube import util, html_common, yt_data_extract, proto
+
 import json
 import urllib
 import html
 from string import Template
 import base64
 from math import ceil
-from youtube.common import default_multi_get, get_thumbnail_url, URL_ORIGIN
-from youtube import common, proto
+
 
 with open("yt_search_results_template.html", "r") as file:
     yt_search_results_template = file.read()
@@ -54,7 +55,7 @@ def get_search_json(query, page, autocorrect, sort, filters):
         'X-YouTube-Client-Version': '2.20180418',
     }
     url += "&pbj=1&sp=" + page_number_to_sp_parameter(page, autocorrect, sort, filters).replace("=", "%3D")
-    content = common.fetch_url(url, headers=headers, report_text="Got search results")
+    content = util.fetch_url(url, headers=headers, report_text="Got search results")
     info = json.loads(content)
     return info
     
@@ -70,9 +71,9 @@ def get_search_page(env, start_response):
     start_response('200 OK', [('Content-type','text/html'),])
     parameters = env['parameters']
     if len(parameters) == 0:
-        return common.yt_basic_template.substitute(
+        return html_common.yt_basic_template.substitute(
             page_title = "Search",
-            header = common.get_header(),
+            header = html_common.get_header(),
             style = '',
             page = '',
         ).encode('utf-8')
@@ -100,24 +101,24 @@ def get_search_page(env, start_response):
             renderer = renderer[type]
             corrected_query_string = parameters.copy()
             corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']]
-            corrected_query_url = URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True)
+            corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True)
             corrections = did_you_mean.substitute(
                 corrected_query_url = corrected_query_url,
-                corrected_query = common.format_text_runs(renderer['correctedQuery']['runs']),
+                corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
             )
             continue
         if type == 'showingResultsForRenderer':
             renderer = renderer[type]
             no_autocorrect_query_string = parameters.copy()
             no_autocorrect_query_string['autocorrect'] = ['0']
-            no_autocorrect_query_url = URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True)
+            no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True)
             corrections = showing_results_for.substitute(
-                corrected_query = common.format_text_runs(renderer['correctedQuery']['runs']),
+                corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']),
                 original_query_url = no_autocorrect_query_url,
                 original_query = html.escape(renderer['originalQuery']['simpleText']),
             )
             continue
-        result_list_html += common.renderer_html(renderer, current_query_string=env['QUERY_STRING'])
+        result_list_html += html_common.renderer_html(renderer, current_query_string=env['QUERY_STRING'])
         
     page = int(page)
     if page <= 5:
@@ -129,13 +130,13 @@ def get_search_page(env, start_response):
         
     
     result = Template(yt_search_results_template).substitute(
-        header              = common.get_header(query),
+        header              = html_common.get_header(query),
         results             = result_list_html, 
         page_title          = query + " - Search", 
         search_box_value    = html.escape(query),
         number_of_results   = '{:,}'.format(estimated_results),
         number_of_pages     = '{:,}'.format(estimated_pages),
-        page_buttons        = common.page_buttons_html(page, estimated_pages, URL_ORIGIN + "/search", env['QUERY_STRING']),
+        page_buttons        = html_common.page_buttons_html(page, estimated_pages, util.URL_ORIGIN + "/search", env['QUERY_STRING']),
         corrections         = corrections
         )
     return result.encode('utf-8')
diff --git a/youtube/subscriptions.py b/youtube/subscriptions.py
index ff7d0df..0c7e8a5 100644
--- a/youtube/subscriptions.py
+++ b/youtube/subscriptions.py
@@ -1,4 +1,4 @@
-from youtube import common, channel
+from youtube import util, yt_data_extract, html_common, channel
 import settings
 from string import Template
 import sqlite3
@@ -169,7 +169,7 @@ def _get_upstream_videos(channel_id, time_last_checked):
 
     content = response.read()
     print('Retrieved videos for ' + channel_id)
-    content = common.decode_content(content, response.getheader('Content-Encoding', default='identity'))
+    content = util.decode_content(content, response.getheader('Content-Encoding', default='identity'))
 
 
     feed = atoma.parse_atom_bytes(content)
@@ -191,7 +191,7 @@ def _get_upstream_videos(channel_id, time_last_checked):
     # Now check channel page to retrieve missing information for videos
     json_channel_videos = channel.get_grid_items(channel.get_channel_tab(channel_id)[1]['response'])
     for json_video in json_channel_videos:
-        info = common.renderer_info(json_video['gridVideoRenderer'])
+        info = yt_data_extract.renderer_info(json_video['gridVideoRenderer'])
         if 'description' not in info:
             info['description'] = ''
         if info['id'] in atom_videos:
@@ -205,12 +205,12 @@ def get_subscriptions_page(env, start_response):
     items_html = '''<nav class="item-grid">\n'''
 
     for item in _get_videos(30, 0):
-        items_html += common.video_item_html(item, common.small_video_item_template)
+        items_html += html_common.video_item_html(item, html_common.small_video_item_template)
     items_html += '''\n</nav>'''
 
     start_response('200 OK', [('Content-type','text/html'),])
     return subscriptions_template.substitute(
-        header = common.get_header(),
+        header = html_common.get_header(),
         items = items_html,
         page_buttons = '',
     ).encode('utf-8')
@@ -243,7 +243,7 @@ def post_subscriptions_page(env, start_response):
         finally:
             connection.close()
 
-        start_response('303 See Other', [('Location', common.URL_ORIGIN + '/subscriptions'),] )
+        start_response('303 See Other', [('Location', util.URL_ORIGIN + '/subscriptions'),] )
         return b''
     else:
         start_response('400 Bad Request', ())
diff --git a/youtube/util.py b/youtube/util.py
new file mode 100644
index 0000000..9950815
--- /dev/null
+++ b/youtube/util.py
@@ -0,0 +1,229 @@
+import settings
+import socks, sockshandler
+import gzip
+import brotli
+import urllib.parse
+import re
+import time
+
+# The trouble with the requests library: It ships its own certificate bundle via certifi
+#  instead of using the system certificate store, meaning self-signed certificates
+#  configured by the user will not work. Some draconian networks block TLS unless a corporate
+#  certificate is installed on the system. Additionally, some users install a self signed cert
+#  in order to use programs to modify or monitor requests made by programs on the system.
+
+# Finally, certificates expire and need to be updated, or are sometimes revoked. Sometimes
+#  certificate authorites go rogue and need to be untrusted. Since we are going through Tor exit nodes,
+#  this becomes all the more important. A rogue CA could issue a fake certificate for accounts.google.com, and a
+#  malicious exit node could use this to decrypt traffic when logging in and retrieve passwords. Examples:
+#   https://www.engadget.com/2015/10/29/google-warns-symantec-over-certificates/
+#   https://nakedsecurity.sophos.com/2013/12/09/serious-security-google-finds-fake-but-trusted-ssl-certificates-for-its-domains-made-in-france/
+
+# In the requests documentation it says:
+#    "Before version 2.16, Requests bundled a set of root CAs that it trusted, sourced from the Mozilla trust store.
+#     The certificates were only updated once for each Requests version. When certifi was not installed,
+#     this led to extremely out-of-date certificate bundles when using significantly older versions of Requests.
+#     For the sake of security we recommend upgrading certifi frequently!"
+#   (http://docs.python-requests.org/en/master/user/advanced/#ca-certificates)
+
+# Expecting users to remember to manually update certifi on Linux isn't reasonable in my view.
+#  On windows, this is even worse since I am distributing all dependencies. This program is not
+#  updated frequently, and using requests would lead to outdated certificates. Certificates
+#  should be updated with OS updates, instead of thousands of developers of different programs
+#  being expected to do this correctly 100% of the time.
+
+# There is hope that this might be fixed eventually:
+#   https://github.com/kennethreitz/requests/issues/2966
+
+# Until then, I will use a mix of urllib3 and urllib.
+import urllib3
+import urllib3.contrib.socks
+
+URL_ORIGIN = "/https://www.youtube.com"
+
+connection_pool = urllib3.PoolManager(cert_reqs = 'CERT_REQUIRED')
+
+old_tor_connection_pool = None
+tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:9150/', cert_reqs = 'CERT_REQUIRED')
+
+tor_pool_refresh_time = time.monotonic()   # prevent problems due to clock changes
+
+def get_pool(use_tor):
+    global old_tor_connection_pool
+    global tor_connection_pool
+    global tor_pool_refresh_time
+
+    if not use_tor:
+        return connection_pool
+
+    # Tor changes circuits after 10 minutes: https://tor.stackexchange.com/questions/262/for-how-long-does-a-circuit-stay-alive
+    current_time = time.monotonic()
+    if current_time - tor_pool_refresh_time > 300:   # close pool after 5 minutes
+        tor_connection_pool.clear()
+
+        # Keep a reference for 5 min to avoid it getting garbage collected while sockets still in use
+        old_tor_connection_pool = tor_connection_pool
+
+        tor_connection_pool = urllib3.contrib.socks.SOCKSProxyManager('socks5://127.0.0.1:9150/', cert_reqs = 'CERT_REQUIRED')
+        tor_pool_refresh_time = current_time
+
+    return tor_connection_pool
+
+
+
+class HTTPAsymmetricCookieProcessor(urllib.request.BaseHandler):
+    '''Separate cookiejars for receiving and sending'''
+    def __init__(self, cookiejar_send=None, cookiejar_receive=None):
+        import http.cookiejar
+        self.cookiejar_send = cookiejar_send
+        self.cookiejar_receive = cookiejar_receive
+
+    def http_request(self, request):
+        if self.cookiejar_send is not None:
+            self.cookiejar_send.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        if self.cookiejar_receive is not None:
+            self.cookiejar_receive.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+
+def decode_content(content, encoding_header):
+    encodings = encoding_header.replace(' ', '').split(',')
+    for encoding in reversed(encodings):
+        if encoding == 'identity':
+            continue
+        if encoding == 'br':
+            content = brotli.decompress(content)
+        elif encoding == 'gzip':
+            content = gzip.decompress(content)
+    return content
+
+def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, cookiejar_send=None, cookiejar_receive=None, use_tor=True, return_response=False):
+    '''
+    When cookiejar_send is set to a CookieJar object,
+     those cookies will be sent in the request (but cookies in response will not be merged into it)
+    When cookiejar_receive is set to a CookieJar object,
+     cookies received in the response will be merged into the object (nothing will be sent from it)
+    When both are set to the same object, cookies will be sent from the object,
+     and response cookies will be merged into it.
+    '''
+    headers = dict(headers)     # Note: Calling dict() on a dict will make a copy
+    headers['Accept-Encoding'] = 'gzip, br'
+
+    # prevent python version being leaked by urllib if User-Agent isn't provided
+    #  (urllib will use ex. Python-urllib/3.6 otherwise)
+    if 'User-Agent' not in headers and 'user-agent' not in headers and 'User-agent' not in headers:
+        headers['User-Agent'] = 'Python-urllib'
+
+    method = "GET"
+    if data is not None:
+        method = "POST"
+        if isinstance(data, str):
+            data = data.encode('ascii')
+        elif not isinstance(data, bytes):
+            data = urllib.parse.urlencode(data).encode('ascii')
+
+    start_time = time.time()
+
+    if cookiejar_send is not None or cookiejar_receive is not None:     # Use urllib
+        req = urllib.request.Request(url, data=data, headers=headers)
+
+        cookie_processor = HTTPAsymmetricCookieProcessor(cookiejar_send=cookiejar_send, cookiejar_receive=cookiejar_receive)
+
+        if use_tor and settings.route_tor:
+            opener = urllib.request.build_opener(sockshandler.SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150), cookie_processor)
+        else:
+            opener = urllib.request.build_opener(cookie_processor)
+
+        response = opener.open(req, timeout=timeout)
+        response_time = time.time()
+
+
+        content = response.read()
+
+    else:           # Use a urllib3 pool. Cookies can't be used since urllib3 doesn't have easy support for them.
+        pool = get_pool(use_tor and settings.route_tor)
+
+        response = pool.request(method, url, headers=headers, timeout=timeout, preload_content=False, decode_content=False)
+        response_time = time.time()
+
+        content = response.read()
+        response.release_conn()
+
+    read_finish = time.time()
+    if report_text:
+        print(report_text, '    Latency:', round(response_time - start_time,3), '    Read time:', round(read_finish - response_time,3))
+    content = decode_content(content, response.getheader('Content-Encoding', default='identity'))
+
+    if return_response:
+        return content, response
+    return content
+
+mobile_user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1'
+mobile_ua = (('User-Agent', mobile_user_agent),)
+desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0'
+desktop_ua = (('User-Agent', desktop_user_agent),)
+
+
+
+
+
+
+
+
+
+
+def dict_add(*dicts):
+    for dictionary in dicts[1:]:
+        dicts[0].update(dictionary)
+    return dicts[0]
+
+def video_id(url):
+    url_parts = urllib.parse.urlparse(url)
+    return urllib.parse.parse_qs(url_parts.query)['v'][0]
+
+def default_multi_get(object, *keys, default):
+    ''' Like dict.get(), but for nested dictionaries/sequences, supporting keys or indices. Last argument is the default value to use in case of any IndexErrors or KeyErrors '''
+    try:
+        for key in keys:
+            object = object[key]
+        return object
+    except (IndexError, KeyError):
+        return default
+
+
+# default, sddefault, mqdefault, hqdefault, hq720
+def get_thumbnail_url(video_id):
+    return "/i.ytimg.com/vi/" + video_id + "/mqdefault.jpg"
+    
+def seconds_to_timestamp(seconds):
+    seconds = int(seconds)
+    hours, seconds = divmod(seconds,3600)
+    minutes, seconds = divmod(seconds,60)
+    if hours != 0:
+        timestamp = str(hours) + ":"
+        timestamp += str(minutes).zfill(2)  # zfill pads with zeros
+    else:
+        timestamp = str(minutes)
+
+    timestamp += ":" + str(seconds).zfill(2)
+    return timestamp
+
+
+
+def update_query_string(query_string, items):
+    parameters = urllib.parse.parse_qs(query_string)
+    parameters.update(items)
+    return urllib.parse.urlencode(parameters, doseq=True)
+
+
+
+def uppercase_escape(s):
+     return re.sub(
+         r'\\U([0-9a-fA-F]{8})',
+         lambda m: chr(int(m.group(1), base=16)), s)
\ No newline at end of file
diff --git a/youtube/watch.py b/youtube/watch.py
index 04a5b5d..06b525a 100644
--- a/youtube/watch.py
+++ b/youtube/watch.py
@@ -1,12 +1,12 @@
+from youtube import util, html_common, comments
+
 from youtube_dl.YoutubeDL import YoutubeDL
 from youtube_dl.extractor.youtube import YoutubeError
 import json
 import urllib
 from string import Template
 import html
-import youtube.common as common
-from youtube.common import default_multi_get, get_thumbnail_url, video_id, URL_ORIGIN
-import youtube.comments as comments
+
 import gevent
 import settings
 import os
@@ -127,9 +127,11 @@ def get_related_items_html(info):
     result = ""
     for item in info['related_vids']:
         if 'list' in item:  # playlist:
-            result += common.small_playlist_item_html(watch_page_related_playlist_info(item))
+            item = watch_page_related_playlist_info(item)
+            result += html_common.playlist_item_html(item, html_common.small_playlist_item_template)
         else:
-            result += common.small_video_item_html(watch_page_related_video_info(item))
+            item = watch_page_related_video_info(item)
+            result += html_common.video_item_html(item, html_common.small_video_item_template)
     return result
 
     
@@ -137,11 +139,12 @@ def get_related_items_html(info):
 # converts these to standard names
 def watch_page_related_video_info(item):
     result = {key: item[key] for key in ('id', 'title', 'author')}
-    result['duration'] = common.seconds_to_timestamp(item['length_seconds'])
+    result['duration'] = util.seconds_to_timestamp(item['length_seconds'])
     try:
         result['views'] = item['short_view_count_text']
     except KeyError:
         result['views'] = ''
+    result['thumbnail'] = util.get_thumbnail_url(item['id'])
     return result
     
 def watch_page_related_playlist_info(item):
@@ -150,14 +153,15 @@ def watch_page_related_playlist_info(item):
         'title': item['playlist_title'],
         'id': item['list'],
         'first_video_id': item['video_id'],
+        'thumbnail': util.get_thumbnail_url(item['video_id']),
     }
 
     
 def sort_formats(info):
     sorted_formats = info['formats'].copy()
-    sorted_formats.sort(key=lambda x: default_multi_get(_formats, x['format_id'], 'height', default=0))
+    sorted_formats.sort(key=lambda x: util.default_multi_get(_formats, x['format_id'], 'height', default=0))
     for index, format in enumerate(sorted_formats):
-        if default_multi_get(_formats, format['format_id'], 'height', default=0) >= 360:
+        if util.default_multi_get(_formats, format['format_id'], 'height', default=0) >= 360:
             break
     sorted_formats = sorted_formats[index:] + sorted_formats[0:index]
     sorted_formats = [format for format in info['formats'] if format['acodec'] != 'none' and format['vcodec'] != 'none']
@@ -236,7 +240,7 @@ def get_watch_page(env, start_response):
 
         start_response('200 OK', [('Content-type','text/html'),])
 
-        lc = common.default_multi_get(env['parameters'], 'lc', 0, default='')
+        lc = util.default_multi_get(env['parameters'], 'lc', 0, default='')
         if settings.route_tor:
             proxy = 'socks5://127.0.0.1:9150/'
         else:
@@ -256,17 +260,17 @@ def get_watch_page(env, start_response):
         #chosen_format = choose_format(info)
 
         if isinstance(info, str): # youtube error
-            return common.yt_basic_template.substitute(
+            return html_common.yt_basic_template.substitute(
                 page_title = "Error",
                 style = "",
-                header = common.get_header(),
+                header = html_common.get_header(),
                 page = html.escape(info),
             ).encode('utf-8')
             
         sorted_formats = sort_formats(info)
         
         video_info = {
-            "duration": common.seconds_to_timestamp(info["duration"]),
+            "duration": util.seconds_to_timestamp(info["duration"]),
             "id":       info['id'],
             "title":    info['title'],
             "author":   info['uploader'],
@@ -338,7 +342,7 @@ def get_watch_page(env, start_response):
         page = yt_watch_template.substitute(
             video_title             = html.escape(info["title"]),
             page_title              = html.escape(info["title"]),
-            header                  = common.get_header(),
+            header                  = html_common.get_header(),
             uploader                = html.escape(info["uploader"]),
             uploader_channel_url    = '/' + info["uploader_url"],
             upload_date             = upload_date,
diff --git a/youtube/youtube.py b/youtube/youtube.py
index 288f68b..4ec7962 100644
--- a/youtube/youtube.py
+++ b/youtube/youtube.py
@@ -1,7 +1,7 @@
 import mimetypes
 import urllib.parse
 import os
-from youtube import local_playlist, watch, search, playlist, channel, comments, common, post_comment, accounts, subscriptions
+from youtube import local_playlist, watch, search, playlist, channel, comments, post_comment, accounts, util, subscriptions
 import settings
 YOUTUBE_FILES = (
     "/shared.css",
@@ -68,7 +68,7 @@ def youtube(env, start_response):
 
         elif path.startswith("/api/"):
             start_response('200 OK',  [('Content-type', 'text/vtt'),] )
-            result = common.fetch_url('https://www.youtube.com' + path + ('?' + query_string if query_string else ''))
+            result = util.fetch_url('https://www.youtube.com' + path + ('?' + query_string if query_string else ''))
             result = result.replace(b"align:start position:0%", b"")
             return result
 
diff --git a/youtube/yt_data_extract.py b/youtube/yt_data_extract.py
new file mode 100644
index 0000000..5483911
--- /dev/null
+++ b/youtube/yt_data_extract.py
@@ -0,0 +1,205 @@
+import html
+
+# videos (all of type str):
+
+# id
+# title
+# url
+# author
+# author_url
+# thumbnail
+# description
+# published
+# duration
+# likes
+# dislikes
+# views
+# playlist_index
+
+# playlists:
+
+# id
+# title
+# url
+# author
+# author_url
+# thumbnail
+# description
+# updated
+# size
+# first_video_id
+
+
+
+
+
+
+
+def get_plain_text(node):
+    try:
+        return html.escape(node['simpleText'])
+    except KeyError:
+        return unformmated_text_runs(node['runs'])
+        
+def unformmated_text_runs(runs):
+    result = ''
+    for text_run in runs:
+        result += html.escape(text_run["text"])
+    return result
+
+def format_text_runs(runs):
+    if isinstance(runs, str):
+        return runs
+    result = ''
+    for text_run in runs:
+        if text_run.get("bold", False):
+            result += "<b>" + html.escape(text_run["text"]) + "</b>"
+        elif text_run.get('italics', False):
+            result += "<i>" + html.escape(text_run["text"]) + "</i>"
+        else:
+            result += html.escape(text_run["text"])
+    return result
+
+
+
+
+
+
+
+
+def get_url(node):
+    try:
+        return node['runs'][0]['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
+    except KeyError:
+        return node['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
+
+
+def get_text(node):
+    try:
+        return node['simpleText']
+    except KeyError:
+            pass
+    try:
+        return node['runs'][0]['text']
+    except IndexError: # empty text runs
+        return ''
+
+def get_formatted_text(node):
+    try:
+        return node['runs']
+    except KeyError:
+        return node['simpleText']
+
+def get_badges(node):
+    badges = []
+    for badge_node in node:
+        badge = badge_node['metadataBadgeRenderer']['label']
+        if badge.lower() != 'new':
+            badges.append(badge)
+    return badges
+
+def get_thumbnail(node):
+    try:
+        return node['thumbnails'][0]['url']     # polymer format
+    except KeyError:
+        return node['url']     # ajax format
+
+dispatch = {
+
+# polymer format    
+    'title':                ('title',       get_text),
+    'publishedTimeText':    ('published',   get_text),
+    'videoId':              ('id',          lambda node: node),
+    'descriptionSnippet':   ('description', get_formatted_text),
+    'lengthText':           ('duration',    get_text),
+    'thumbnail':            ('thumbnail',   get_thumbnail),
+    'thumbnails':           ('thumbnail',   lambda node: node[0]['thumbnails'][0]['url']),
+
+    'viewCountText':        ('views',       get_text),
+    'numVideosText':        ('size',        lambda node: get_text(node).split(' ')[0]),     # the format is "324 videos"
+    'videoCountText':       ('size',        get_text),
+    'playlistId':           ('id',          lambda node: node),
+    'descriptionText':      ('description', get_formatted_text),
+
+    'subscriberCountText':  ('subscriber_count',    get_text),
+    'channelId':            ('id',          lambda node: node),
+    'badges':               ('badges',      get_badges),
+
+# ajax format
+    'view_count_text':  ('views',       get_text),
+    'num_videos_text':  ('size',        lambda node: get_text(node).split(' ')[0]),
+    'owner_text':       ('author',      get_text),
+    'owner_endpoint':   ('author_url',  lambda node: node['url']),
+    'description':      ('description', get_formatted_text),
+    'index':            ('playlist_index', get_text),
+    'short_byline':     ('author',      get_text),
+    'length':           ('duration',    get_text),
+    'video_id':         ('id',          lambda node: node),
+
+}
+
+def renderer_info(renderer):
+    try:
+        info = {}
+        if 'viewCountText' in renderer:     # prefer this one as it contains all the digits
+            info['views'] = get_text(renderer['viewCountText'])
+        elif 'shortViewCountText' in renderer:
+            info['views'] = get_text(renderer['shortViewCountText'])
+
+        if 'ownerText' in renderer:
+            info['author'] = renderer['ownerText']['runs'][0]['text']
+            info['author_url'] = renderer['ownerText']['runs'][0]['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']
+        try:
+            overlays = renderer['thumbnailOverlays']
+        except KeyError:
+            pass
+        else:
+            for overlay in overlays:
+                if 'thumbnailOverlayTimeStatusRenderer' in overlay:
+                    info['duration'] = get_text(overlay['thumbnailOverlayTimeStatusRenderer']['text'])
+                # show renderers don't have videoCountText
+                elif 'thumbnailOverlayBottomPanelRenderer' in overlay:
+                    info['size'] = get_text(overlay['thumbnailOverlayBottomPanelRenderer']['text'])
+
+        # show renderers don't have playlistId, have to dig into the url to get it
+        try:
+            info['id'] = renderer['navigationEndpoint']['watchEndpoint']['playlistId']
+        except KeyError:
+            pass
+        for key, node in renderer.items():
+            if key in ('longBylineText', 'shortBylineText'):
+                info['author'] = get_text(node)
+                try:
+                    info['author_url'] = get_url(node)
+                except KeyError:
+                    pass
+
+            # show renderers don't have thumbnail key at top level, dig into thumbnailRenderer
+            elif key == 'thumbnailRenderer' and 'showCustomThumbnailRenderer' in node:
+                info['thumbnail'] = node['showCustomThumbnailRenderer']['thumbnail']['thumbnails'][0]['url']
+            else:
+                try:
+                    simple_key, function = dispatch[key]
+                except KeyError:
+                    continue
+                info[simple_key] = function(node)
+        return info
+    except KeyError:
+        print(renderer)
+        raise
+    
+def ajax_info(item_json):
+    try:
+        info = {}          
+        for key, node in item_json.items():
+            try:
+                simple_key, function = dispatch[key]
+            except KeyError:
+                continue
+            info[simple_key] = function(node)
+        return info
+    except KeyError:
+        print(item_json)
+        raise
+    
+