Merge branch 'modular-data-extract'

Commits in this branch are prefixed with "Extraction:"
This branch refactors data extraction. All such functionality has been moved to the yt_data_extract module.
Responses from requests are given to the module and it parses them into a consistent, more useful format.
The dependency on youtube-dl has also been dropped and this functionality has been built from scratch for these reasons:
(1) I've noticed youtube-dl breaks more often than invidious (which uses watch page extraction built from scratch) in response to changes from Youtube, so I'm hoping what I wrote will also be less brittle.
(2) Such breakage is inconvenient because I have to manually merge the fixes since I had to make changes to youtube-dl to make it do things such as extracting related videos.
(3) I have no control over error handling and request pooling with youtube-dl, since it does all the requests (these would require intrusive changes I don't want to maintain).
(4) I will now be able to finally display the number of comments and whether comments are disabled without making additional requests.

5 files changed, 128 insertions, 72 deletions

diff --git a/youtube/templates/comments.html b/youtube/templates/comments.html
index 20cde4e..396852a 100644
--- a/youtube/templates/comments.html
+++ b/youtube/templates/comments.html
@@ -12,11 +12,11 @@
                 <a class="author" href="{{ comment['author_url'] }}" title="{{ comment['author'] }}">{{ comment['author'] }}</a>
             </address>
             <a class="permalink" href="{{ comment['permalink'] }}" title="permalink">
-                <time datetime="">{{ comment['published'] }}</time>
+                <time datetime="">{{ comment['time_published'] }}</time>
             </a>
             <span class="text">{{ common_elements.text_runs(comment['text']) }}</span>
 
-            <span class="likes">{{ comment['likes_text'] if comment['likes'] else ''}}</span>
+            <span class="likes">{{ comment['likes_text'] if comment['like_count'] else ''}}</span>
             <div class="bottom-row">
                 <a href="{{ comment['replies_url'] }}" class="replies">{{ comment['view_replies_text'] }}</a>
                 {% if 'delete_url' is in comment %}
diff --git a/youtube/templates/common_elements.html b/youtube/templates/common_elements.html
index 67655b3..7914c08 100644
--- a/youtube/templates/common_elements.html
+++ b/youtube/templates/common_elements.html
@@ -9,53 +9,59 @@
                 {{ text_run["text"] }}
             {%- endif -%}
         {%- endfor -%}
-    {%- else -%}
+    {%- elif runs -%}
         {{ runs }}
     {%- endif -%}
 {% endmacro %}
 
-{% macro item(info, description=false, horizontal=true, include_author=true) %}
+{% macro item(info, description=false, horizontal=true, include_author=true, include_badges=true) %}
     <div class="item-box {{ info['type'] + '-item-box' }} {{'horizontal-item-box' if horizontal else 'vertical-item-box'}} {{'has-description' if description else 'no-description'}}">
-        <div class="item {{ info['type'] + '-item' }}">
-            <a class="thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
-                <img class="thumbnail-img" src="{{ info['thumbnail'] }}">
-                {% if info['type'] != 'channel' %}
-                    <div class="thumbnail-info">
-                        <span>{{ info['size'] if info['type'] == 'playlist' else info['duration'] }}</span>
-                    </div>
-                {% endif %}
-            </a>
+        {% if info['error'] %}
+            {{ info['error'] }}
+        {% else %}
+            <div class="item {{ info['type'] + '-item' }}">
+                <a class="thumbnail-box" href="{{ info['url'] }}" title="{{ info['title'] }}">
+                    <img class="thumbnail-img" src="{{ info['thumbnail'] }}">
+                    {% if info['type'] != 'channel' %}
+                        <div class="thumbnail-info">
+                            <span>{{ (info['video_count']|string + ' videos') if info['type'] == 'playlist' else info['duration'] }}</span>
+                        </div>
+                    {% endif %}
+                </a>
 
-            <div class="title"><a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a></div>
+                <div class="title"><a class="title" href="{{ info['url'] }}" title="{{ info['title'] }}">{{ info['title'] }}</a></div>
 
-            <ul class="stats {{'vertical-stats' if horizontal and not description and include_author else 'horizontal-stats'}}">
-                {% if info['type'] == 'channel' %}
-                    <li><span>{{ info['subscriber_count'] }} subscribers</span></li>
-                    <li><span>{{ info['size'] }} videos</span></li>
-                {% else %}
-                    {% if include_author %}
-                        {% if 'author_url' is in(info) %}
-                            <li><address title="{{ info['author'] }}">By <a href="{{ info['author_url'] }}">{{ info['author'] }}</a></address></li>
-                        {% else %}
-                            <li><address title="{{ info['author'] }}"><b>{{ info['author'] }}</b></address></li>
-                        {% endif %}
-                    {% endif %}
-                    {% if 'views' is in(info) %}
-                        <li><span class="views">{{ info['views'] }}</span></li>
-                    {% endif %}
-                    {% if 'published' is in(info) %}
-                        <li><time>{{ info['published'] }}</time></li>
+                {% if include_author %}
+                    {% if info.get('author_url') %}
+                        <address title="{{ info['author'] }}">By <a href="{{ info['author_url'] }}">{{ info['author'] }}</a></address>
+                    {% else %}
+                        <address title="{{ info['author'] }}"><b>{{ info['author'] }}</b></address>
                     {% endif %}
                 {% endif %}
-            </ul>
+                <ul class="stats {{'horizontal-stats' if horizontal else 'vertical-stats'}}">
+                    {% if info['type'] == 'channel' %}
+                        <li><span>{{ info['approx_subscriber_count'] }} subscribers</span></li>
+                        <li><span>{{ info['video_count'] }} videos</span></li>
+                    {% else %}
+                        {% if info.get('approx_view_count') %}
+                            <li><span class="views">{{ info['approx_view_count'] }} views</span></li>
+                        {% endif %}
+                        {% if info.get('time_published') %}
+                            <li><time>{{ info['time_published'] }}</time></li>
+                        {% endif %}
+                    {% endif %}
+                </ul>
 
-            {% if description %}
-                <span class="description">{{ text_runs(info.get('description', '')) }}</span>
+                {% if description %}
+                    <span class="description">{{ text_runs(info.get('description', '')) }}</span>
+                {% endif %}
+                {% if include_badges %}
+                    <span class="badges">{{ info['badges']|join(' | ') }}</span>
+                {% endif %}
+            </div>
+            {% if info['type'] == 'video' %}
+                <input class="item-checkbox" type="checkbox" name="video_info_list" value="{{ info['video_info'] }}" form="playlist-edit">
             {% endif %}
-            <span class="badges">{{ info['badges']|join(' | ') }}</span>
-        </div>
-        {% if info['type'] == 'video' %}
-            <input class="item-checkbox" type="checkbox" name="video_info_list" value="{{ info['video_info'] }}" form="playlist-edit">
         {% endif %}
     </div>
 
diff --git a/youtube/templates/playlist.html b/youtube/templates/playlist.html
index ab2640f..ebd152b 100644
--- a/youtube/templates/playlist.html
+++ b/youtube/templates/playlist.html
@@ -54,8 +54,9 @@
         <h2 class="playlist-title">{{ title }}</h2>
         <a class="playlist-author" href="{{ author_url }}">{{ author }}</a>
         <div class="playlist-stats">
-            <div>{{ views }}</div>
-            <div>{{ size }}</div>
+            <div>{{ video_count|commatize }} videos</div>
+            <div>{{ view_count|commatize }} views</div>
+            <div>Last updated {{ time_published }}</div>
         </div>
         <div class="playlist-description">{{ common_elements.text_runs(description) }}</div>
     </div>
diff --git a/youtube/templates/search.html b/youtube/templates/search.html
index aef914a..8b803e7 100644
--- a/youtube/templates/search.html
+++ b/youtube/templates/search.html
@@ -29,10 +29,10 @@
             <div id="result-info">
                 <div id="number-of-results">Approximately {{ '{:,}'.format(estimated_results) }} results ({{ '{:,}'.format(estimated_pages) }} pages)</div>
 {% if corrections['type'] == 'showing_results_for' %}
-                <div>Showing results for <a>{{ corrections['corrected_query']|safe }}</a></div>
-                <div>Search instead for <a href="{{ corrections['original_query_url'] }}">{{ corrections['original_query'] }}</a></div>
+                <div>Showing results for <a>{{ common_elements.text_runs(corrections['corrected_query_text']) }}</a></div>
+                <div>Search instead for <a href="{{ corrections['original_query_url'] }}">{{ corrections['original_query_text'] }}</a></div>
 {% elif corrections['type'] == 'did_you_mean' %}
-                <div>Did you mean <a href="{{ corrections['corrected_query_url'] }}">{{ corrections['corrected_query']|safe }}</a></div>
+                <div>Did you mean <a href="{{ corrections['corrected_query_url'] }}">{{ common_elements.text_runs(corrections['corrected_query_text']) }}</a></div>
 {% endif %}
             </div>
             <div class="item-list">
diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html
index 25bd34c..a06e895 100644
--- a/youtube/templates/watch.html
+++ b/youtube/templates/watch.html
@@ -14,6 +14,19 @@
         text-decoration: underline;
     }
 
+    .playability-error{
+        height: 360px;
+        width: 640px;
+        grid-column: 2;
+        background-color: var(--video-background-color);
+        text-align:center;
+    }
+        .playability-error span{
+            position: relative;
+            top: 50%;
+            transform: translate(-50%, -50%);
+        }
+
     {% if theater_mode %}
         video{
             grid-column: 1 / span 5;
@@ -61,12 +74,21 @@
                 grid-column: 1 / span 2;
                 min-width: 0;
             }
-            .video-info > .is-unlisted{
-                background-color: var(--interface-color);
+            .video-info > .labels{
                 justify-self:start;
-                padding-left:2px;
-                padding-right:2px;
+                list-style: none;
+                padding: 0px;
+                margin: 5px 0px;
+            }
+            .video-info > .labels:empty{
+                margin: 0px;
             }
+                .labels > li{
+                    display: inline;
+                    margin-right:5px;
+                    background-color: var(--interface-color);
+                    padding: 2px 5px
+                }
             .video-info > address{
                 grid-column: 1;
                 grid-row: 3;
@@ -143,9 +165,13 @@
         .related-videos-inner{
             padding-top: 10px;
             display: grid;
-            grid-auto-rows: 94px;
+            grid-auto-rows: 90px;
             grid-row-gap: 10px;
         }
+            .thumbnail-box{     /* overides rule in shared.css */
+                height: 90px !important;
+                width: 120px !important;
+            }
 
     /* Put related vids below videos when window is too small */
     /* 1100px instead of 1080 because W3C is full of idiots who include scrollbar width */
@@ -187,38 +213,59 @@
                 .format-ext{
                     width: 60px;
                 }
-                .format-res{
-                    width:90px;
+                .format-video-quality{
+                    width: 140px;
+                }
+                .format-audio-quality{
+                    width: 120px;
+                }
+                .format-file-size{
+                    width: 80px;
+                }
+                .format-codecs{
+                    width: 120px;
                 }
 {% endblock style %}
 
 {% block main %}
-    <video controls autofocus>
-        {% for video_source in video_sources %}
-            <source src="{{ video_source['src'] }}" type="{{ video_source['type'] }}">
-        {% endfor %}
+    {% if playability_error %}
+        <div class="playability-error"><span>{{ 'Error: ' + playability_error }}</span></div>
+    {% else %}
+        <video controls autofocus class="video">
+            {% for video_source in video_sources %}
+                <source src="{{ video_source['src'] }}" type="{{ video_source['type'] }}">
+            {% endfor %}
 
-        {% for source in subtitle_sources %}
-            {% if source['on'] %}
-                <track label="{{ source['label'] }}" src="{{ source['url'] }}" kind="subtitles" srclang="{{ source['srclang'] }}" default>
-            {% else %}
-                <track label="{{ source['label'] }}" src="{{ source['url'] }}" kind="subtitles" srclang="{{ source['srclang'] }}">
-            {% endif %}
-        {% endfor %}
+            {% for source in subtitle_sources %}
+                {% if source['on'] %}
+                    <track label="{{ source['label'] }}" src="{{ source['url'] }}" kind="subtitles" srclang="{{ source['srclang'] }}" default>
+                {% else %}
+                    <track label="{{ source['label'] }}" src="{{ source['url'] }}" kind="subtitles" srclang="{{ source['srclang'] }}">
+                {% endif %}
+            {% endfor %}
 
-    </video>
+        </video>
+    {% endif %}
 
     <div class="video-info">
         <h2 class="title">{{ title }}</h2>
-        {% if unlisted %}
-            <span class="is-unlisted">Unlisted</span>
-        {% endif %}
+        <ul class="labels">
+            {%- if unlisted -%}
+                <li class="is-unlisted">Unlisted</li>
+            {%- endif -%}
+            {%- if age_restricted -%}
+                <li class="age-restricted">Age-restricted</li>
+            {%- endif -%}
+            {%- if limited_state -%}
+                <li>Limited state</li>
+            {%- endif -%}
+        </ul>
         <address>Uploaded by <a href="{{ uploader_channel_url }}">{{ uploader }}</a></address>
-        <span class="views">{{ views }} views</span>
+        <span class="views">{{ view_count }} views</span>
 
 
-        <time datetime="$upload_date">Published on {{ upload_date }}</time>
-        <span class="likes-dislikes">{{ likes }} likes {{ dislikes }} dislikes</span>
+        <time datetime="$upload_date">Published on {{ time_published }}</time>
+        <span class="likes-dislikes">{{ like_count }} likes {{ dislike_count }} dislikes</span>
         <details class="download-dropdown">
             <summary class="download-dropdown-label">Download</summary>
             <ul class="download-dropdown-content">
@@ -227,8 +274,10 @@
                         <a class="download-link" href="{{ format['url'] }}">
                             <ol class="format-attributes">
                                 <li class="format-ext">{{ format['ext'] }}</li>
-                                <li class="format-res">{{ format['resolution'] }}</li>
-                                <li class="format-note">{{ format['note'] }}</li>
+                                <li class="format-video-quality">{{ format['video_quality'] }}</li>
+                                <li class="format-audio-quality">{{ format['audio_quality'] }}</li>
+                                <li class="format-file-size">{{ format['file_size'] }}</li>
+                                <li class="format-codecs">{{ format['codecs'] }}</li>
                             </ol>
                         </a>
                     </li>
@@ -238,7 +287,7 @@
         <input class="checkbox" name="video_info_list" value="{{ video_info }}" form="playlist-edit" type="checkbox">
 
 
-        <span class="description">{{ description }}</span>
+        <span class="description">{{ common_elements.text_runs(description) }}</span>
         <div class="music-list">
             {% if music_list.__len__() != 0 %}
                 <hr>
@@ -266,7 +315,7 @@
             <summary>Related Videos</summary>
             <nav class="related-videos-inner">
                 {% for info in related %}
-                    {{ common_elements.item(info) }}
+                    {{ common_elements.item(info, include_badges=false) }}
                 {% endfor %}
             </nav>
         </details>