From 2db58930a6f8c955c4d437657bd07e2939a705f2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 16 Jun 2019 16:16:03 -0700 Subject: Convert watch page to flask framework --- youtube/__init__.py | 2 + youtube/comments.css | 129 ------------ youtube/favicon.ico | Bin 5694 -> 0 bytes youtube/shared.css | 372 ----------------------------------- youtube/static/comments.css | 129 ++++++++++++ youtube/static/favicon.ico | Bin 0 -> 5694 bytes youtube/static/shared.css | 372 +++++++++++++++++++++++++++++++++++ youtube/templates/error.html | 17 ++ youtube/templates/watch.html | 222 +++++++++++++++++++++ youtube/watch.py | 458 ++++++++++++++++--------------------------- youtube/youtube.py | 105 ---------- 11 files changed, 909 insertions(+), 897 deletions(-) create mode 100644 youtube/__init__.py delete mode 100644 youtube/comments.css delete mode 100644 youtube/favicon.ico delete mode 100644 youtube/shared.css create mode 100644 youtube/static/comments.css create mode 100644 youtube/static/favicon.ico create mode 100644 youtube/static/shared.css create mode 100644 youtube/templates/error.html create mode 100644 youtube/templates/watch.html delete mode 100644 youtube/youtube.py (limited to 'youtube') diff --git a/youtube/__init__.py b/youtube/__init__.py new file mode 100644 index 0000000..0df56d1 --- /dev/null +++ b/youtube/__init__.py @@ -0,0 +1,2 @@ +import flask +yt_app = flask.Flask(__name__) \ No newline at end of file diff --git a/youtube/comments.css b/youtube/comments.css deleted file mode 100644 index 4cec3e1..0000000 --- a/youtube/comments.css +++ /dev/null @@ -1,129 +0,0 @@ -.video-metadata{ - display: grid; - grid-template-columns: auto 1fr; - grid-template-rows: auto auto 1fr auto; -} - .video-metadata > .video-metadata-thumbnail-box{ - grid-row: 1 / span 3; - } - .video-metadata > .title{ - word-wrap:break-word; - grid-row: 1; - } - .video-metadata > h2{ - grid-row: 2; - font-size: 15px; - } - .video-metadata > span{ - grid-row:3; - } - .video-metadata > hr{ - grid-row: 4; - grid-column: 1 / span 2; - width: 100%; - } - -.comment-form{ - display: grid; - align-content: start; - justify-items: start; - align-items: start; -} - #comment-account-options{ - display:grid; - grid-auto-flow: column; - grid-column-gap: 10px; - margin-top:10px; - margin-bottom:10px; - } - #comment-account-options a{ - margin-left:10px; - } - -.comments-area{ - display:grid; -} - .comments-area textarea{ - resize: vertical; - justify-self:stretch; - } - .post-comment-button{ - margin-top:10px; - justify-self:end; - } - .comment-links{ - display:grid; - grid-auto-flow: column; - grid-column-gap: 10px; - justify-content:start; - } - -.comments{ - margin-top:10px; - grid-row-gap: 10px; - display: grid; - align-content:start; -} - -.comment{ - display:grid; - grid-template-columns: auto auto 100px 1fr; - grid-template-rows: 0fr 0fr 0fr 0fr; - background-color: #dadada; - justify-content: start; -} - -.comment .author-avatar{ - grid-column: 1; - grid-row: 1 / span 3; - align-self: start; - margin-right: 5px; - height:32px; - width:32px; -} - -.comment address{ - grid-column: 2; - grid-row: 1; - margin-right:15px; - white-space: nowrap; - overflow:hidden; -} - -.comment .text{ - grid-column: 2 / span 3; - grid-row: 2; - white-space: pre-wrap; - min-width: 0; - word-wrap: break-word; -} - -.comment .permalink{ - grid-column: 3; - grid-row: 1; - white-space: nowrap; - color: black; - -} - - -.comment .likes{ - grid-column:2; - grid-row:3; - font-weight:bold; - white-space: nowrap; -} - -.comment .bottom-row{ - grid-column:2 / span 3; - grid-row:4; - justify-self:start; - display: grid; - grid-auto-flow: column; - grid-column-gap: 10px; -} - -.more-comments{ - justify-self:center; - margin-top:10px; -} diff --git a/youtube/favicon.ico b/youtube/favicon.ico deleted file mode 100644 index 9d6417c..0000000 Binary files a/youtube/favicon.ico and /dev/null differ diff --git a/youtube/shared.css b/youtube/shared.css deleted file mode 100644 index 1b25d7f..0000000 --- a/youtube/shared.css +++ /dev/null @@ -1,372 +0,0 @@ -h1, h2, h3, h4, h5, h6, div, button{ - margin:0; - padding:0; - -} - - -body{ - margin:0; - padding: 0; - color:#222; - - - background-color:#cccccc; - - min-height:100vh; - - display:grid; - grid-template-rows: 50px 1fr; -} - - header{ - background-color:#333333; - - grid-row: 1; - - display:grid; - grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); - } - - main{ - grid-row: 2; - } - -address{ - font-style:normal; -} - - - - #site-search{ - grid-column: 2; - display: grid; - grid-template-columns: 1fr auto auto; - - } - - #site-search .search-box{ - align-self:center; - height:25px; - border:0; - - grid-column: 1; - } - #site-search .search-button{ - grid-column: 2; - align-self:center; - height:25px; - - border-style:solid; - border-width:1px; - } - #site-search .dropdown{ - margin-left:5px; - grid-column: 3; - align-self:center; - height:25px; - } - #site-search .dropdown button{ - align-self:center; - height:25px; - - border-style:solid; - border-width:1px; - } - #site-search .css-sucks{ - width:0px; - height:0px; - } - #site-search .dropdown-content{ - grid-template-columns: auto auto; - white-space: nowrap; - } - #site-search .dropdown-content h3{ - grid-column:1 / span 2; - } - -.dropdown{ - z-index:1; -} - .dropdown-content{ - display:none; - background-color: #e9e9e9; - } - .dropdown:hover .dropdown-content{ - /* For some reason, if this is just grid, it will insist on being 0px wide just like its 0px by 0px parent */ - /* making it inline-grid happened to fix it */ - display:inline-grid; - } - - - -#header-right{ - grid-column:4; - - display:grid; - grid-template-columns:auto auto auto 1fr; - grid-template-rows: 1fr; - width: 540px; -} - #playlist-edit{ - display:contents; - } - #local-playlists{ - grid-column: 1; - grid-row:1; - align-self: center; - margin-right:5px; - color: #ffffff; - } - #playlist-name-selection{ - grid-column:2; - grid-row:1; - justify-self:start; - align-self: center; - } - #playlist-add-button{ - grid-column:3; - grid-row:1; - align-self: center; - padding-left: 10px; - padding-right: 10px; - } - #item-selection-reset{ - grid-column:4; - grid-row:1; - align-self: center; - justify-self:start; - padding-left: 10px; - padding-right: 10px; - } - - - -.item-list{ - display: grid; - grid-auto-rows: 138px; - grid-row-gap: 10px; - -} - .item-list .video-thumbnail-box{ - width:246px; - } - .item-list .playlist-thumbnail-box{ - width:246px; - } - - -.item-grid{ - display:grid; - grid-template-columns: repeat(auto-fill, 400px); - grid-auto-rows: 94px; - grid-row-gap: 10px; -} - .item-grid .video-thumbnail-box{ - width:168px; - } - .item-grid .playlist-thumbnail-box{ - width:168px; - } - - - -.medium-item-box{ - - display:grid; - grid-template-columns: 1fr 30px; -} -.medium-item{ - background-color:#bcbcbc; - text-decoration:none; - font-size: 12px; - color: #767676; - - display: grid; - align-content: start; - grid-template-columns: auto 1fr auto; - grid-template-rows: auto auto auto auto auto 1fr; -} - .medium-item .title{ - grid-column:2 / span 2; - grid-row:1; - justify-self:start; - min-width: 0; - max-height:3.6em; - overflow:hidden; - - color: #333; - font-size: 16px; - font-weight: 500; - text-decoration:initial; - } - .medium-item address{ - display:inline; - } - /*.medium-item .views{ - grid-column: 3; - grid-row: 2; - justify-self:end; - } - .medium-item time{ - grid-column: 2; - grid-row: 3; - justify-self:start; - }*/ - .medium-item .stats{ - grid-column: 2 / span 2; - grid-row: 2; - max-height:2.4em; - overflow:hidden; - } - - .medium-item .description{ - grid-column: 2 / span 2; - grid-row: 4; - } - .medium-item .badges{ - grid-column: 2 / span 2; - grid-row: 5; - } - /* thumbnail size */ - .medium-item img{ - /*height:138px; - width:246px;*/ - height:100%; - justify-self:center; - } - -.small-item-box{ - color: #767676; - font-size: 12px; - - display:grid; - grid-template-columns: 1fr 30px; - grid-template-rows: 94px; -} - -.small-item{ - background-color:#bcbcbc; - align-content: start; - text-decoration:none; - - display: grid; - grid-template-columns: 168px 1fr; - grid-column-gap: 5px; - grid-template-rows: auto auto auto 1fr; -} - .small-item .title{ - grid-column:2; - grid-row:1; - margin:0; - - color: #333; - font-size: 16px; - font-weight: 500; - text-decoration:initial; - min-width: 0; - justify-self:start; - - overflow:hidden; - max-height: 3.3em; - line-height: 1.1em; - } - .small-item address{ - grid-column: 2; - grid-row: 2; - justify-self: start; - } - - .small-item .views{ - grid-column: 2; - grid-row: 3; - justify-self:start; - } - /* thumbnail size */ - .small-item img{ - /*height:94px; - width:168px;*/ - height:100%; - justify-self:center; - } - -.item-checkbox{ - justify-self:start; - align-self:center; - height:30px; - width:30px; - - grid-column: 2; -} - -/* ---Thumbnails for videos---- */ -.video-thumbnail-box{ - max-height:100%; - - grid-column:1; - grid-row:1 / span 6; - - display:grid; - grid-template-columns: 1fr 0fr; -} - .video-thumbnail-img{ - grid-column:1 / span 2; - grid-row:1; - } - .video-duration{ - grid-column: 2; - grid-row: 1; - align-self: end; - opacity: .8; - color: #ffffff; - font-size: 12px; - background-color: #000000; - } - -/* ---Thumbnails for playlists---- */ -.playlist-thumbnail-box{ - max-height:100%; - - grid-column:1; - grid-row:1 / span 6; - - display:grid; - grid-template-columns: 3fr 2fr; -} - .playlist-thumbnail-img{ - grid-column:1 / span 2; - grid-row:1; - } - .playlist-thumbnail-info{ - grid-column:2; - grid-row:1; - - display: grid; - align-items:center; - - text-align:center; - white-space: pre-line; - opacity: .8; - color: #cfcfcf; - background-color: #000000; - } - -.page-button-row{ - justify-self:center; - display: grid; - grid-auto-columns: 40px; - grid-auto-flow: column; - height: 40px; -} - .page-button{ - background-color: #e9e9e9; - border-style: outset; - border-width: 2px; - font-weight: bold; - text-align: center; - } -.sort-button{ - background-color: #d0d0d0; - padding: 2px; - justify-self: start; -} diff --git a/youtube/static/comments.css b/youtube/static/comments.css new file mode 100644 index 0000000..4cec3e1 --- /dev/null +++ b/youtube/static/comments.css @@ -0,0 +1,129 @@ +.video-metadata{ + display: grid; + grid-template-columns: auto 1fr; + grid-template-rows: auto auto 1fr auto; +} + .video-metadata > .video-metadata-thumbnail-box{ + grid-row: 1 / span 3; + } + .video-metadata > .title{ + word-wrap:break-word; + grid-row: 1; + } + .video-metadata > h2{ + grid-row: 2; + font-size: 15px; + } + .video-metadata > span{ + grid-row:3; + } + .video-metadata > hr{ + grid-row: 4; + grid-column: 1 / span 2; + width: 100%; + } + +.comment-form{ + display: grid; + align-content: start; + justify-items: start; + align-items: start; +} + #comment-account-options{ + display:grid; + grid-auto-flow: column; + grid-column-gap: 10px; + margin-top:10px; + margin-bottom:10px; + } + #comment-account-options a{ + margin-left:10px; + } + +.comments-area{ + display:grid; +} + .comments-area textarea{ + resize: vertical; + justify-self:stretch; + } + .post-comment-button{ + margin-top:10px; + justify-self:end; + } + .comment-links{ + display:grid; + grid-auto-flow: column; + grid-column-gap: 10px; + justify-content:start; + } + +.comments{ + margin-top:10px; + grid-row-gap: 10px; + display: grid; + align-content:start; +} + +.comment{ + display:grid; + grid-template-columns: auto auto 100px 1fr; + grid-template-rows: 0fr 0fr 0fr 0fr; + background-color: #dadada; + justify-content: start; +} + +.comment .author-avatar{ + grid-column: 1; + grid-row: 1 / span 3; + align-self: start; + margin-right: 5px; + height:32px; + width:32px; +} + +.comment address{ + grid-column: 2; + grid-row: 1; + margin-right:15px; + white-space: nowrap; + overflow:hidden; +} + +.comment .text{ + grid-column: 2 / span 3; + grid-row: 2; + white-space: pre-wrap; + min-width: 0; + word-wrap: break-word; +} + +.comment .permalink{ + grid-column: 3; + grid-row: 1; + white-space: nowrap; + color: black; + +} + + +.comment .likes{ + grid-column:2; + grid-row:3; + font-weight:bold; + white-space: nowrap; +} + +.comment .bottom-row{ + grid-column:2 / span 3; + grid-row:4; + justify-self:start; + display: grid; + grid-auto-flow: column; + grid-column-gap: 10px; +} + +.more-comments{ + justify-self:center; + margin-top:10px; +} diff --git a/youtube/static/favicon.ico b/youtube/static/favicon.ico new file mode 100644 index 0000000..9d6417c Binary files /dev/null and b/youtube/static/favicon.ico differ diff --git a/youtube/static/shared.css b/youtube/static/shared.css new file mode 100644 index 0000000..1b25d7f --- /dev/null +++ b/youtube/static/shared.css @@ -0,0 +1,372 @@ +h1, h2, h3, h4, h5, h6, div, button{ + margin:0; + padding:0; + +} + + +body{ + margin:0; + padding: 0; + color:#222; + + + background-color:#cccccc; + + min-height:100vh; + + display:grid; + grid-template-rows: 50px 1fr; +} + + header{ + background-color:#333333; + + grid-row: 1; + + display:grid; + grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); + } + + main{ + grid-row: 2; + } + +address{ + font-style:normal; +} + + + + #site-search{ + grid-column: 2; + display: grid; + grid-template-columns: 1fr auto auto; + + } + + #site-search .search-box{ + align-self:center; + height:25px; + border:0; + + grid-column: 1; + } + #site-search .search-button{ + grid-column: 2; + align-self:center; + height:25px; + + border-style:solid; + border-width:1px; + } + #site-search .dropdown{ + margin-left:5px; + grid-column: 3; + align-self:center; + height:25px; + } + #site-search .dropdown button{ + align-self:center; + height:25px; + + border-style:solid; + border-width:1px; + } + #site-search .css-sucks{ + width:0px; + height:0px; + } + #site-search .dropdown-content{ + grid-template-columns: auto auto; + white-space: nowrap; + } + #site-search .dropdown-content h3{ + grid-column:1 / span 2; + } + +.dropdown{ + z-index:1; +} + .dropdown-content{ + display:none; + background-color: #e9e9e9; + } + .dropdown:hover .dropdown-content{ + /* For some reason, if this is just grid, it will insist on being 0px wide just like its 0px by 0px parent */ + /* making it inline-grid happened to fix it */ + display:inline-grid; + } + + + +#header-right{ + grid-column:4; + + display:grid; + grid-template-columns:auto auto auto 1fr; + grid-template-rows: 1fr; + width: 540px; +} + #playlist-edit{ + display:contents; + } + #local-playlists{ + grid-column: 1; + grid-row:1; + align-self: center; + margin-right:5px; + color: #ffffff; + } + #playlist-name-selection{ + grid-column:2; + grid-row:1; + justify-self:start; + align-self: center; + } + #playlist-add-button{ + grid-column:3; + grid-row:1; + align-self: center; + padding-left: 10px; + padding-right: 10px; + } + #item-selection-reset{ + grid-column:4; + grid-row:1; + align-self: center; + justify-self:start; + padding-left: 10px; + padding-right: 10px; + } + + + +.item-list{ + display: grid; + grid-auto-rows: 138px; + grid-row-gap: 10px; + +} + .item-list .video-thumbnail-box{ + width:246px; + } + .item-list .playlist-thumbnail-box{ + width:246px; + } + + +.item-grid{ + display:grid; + grid-template-columns: repeat(auto-fill, 400px); + grid-auto-rows: 94px; + grid-row-gap: 10px; +} + .item-grid .video-thumbnail-box{ + width:168px; + } + .item-grid .playlist-thumbnail-box{ + width:168px; + } + + + +.medium-item-box{ + + display:grid; + grid-template-columns: 1fr 30px; +} +.medium-item{ + background-color:#bcbcbc; + text-decoration:none; + font-size: 12px; + color: #767676; + + display: grid; + align-content: start; + grid-template-columns: auto 1fr auto; + grid-template-rows: auto auto auto auto auto 1fr; +} + .medium-item .title{ + grid-column:2 / span 2; + grid-row:1; + justify-self:start; + min-width: 0; + max-height:3.6em; + overflow:hidden; + + color: #333; + font-size: 16px; + font-weight: 500; + text-decoration:initial; + } + .medium-item address{ + display:inline; + } + /*.medium-item .views{ + grid-column: 3; + grid-row: 2; + justify-self:end; + } + .medium-item time{ + grid-column: 2; + grid-row: 3; + justify-self:start; + }*/ + .medium-item .stats{ + grid-column: 2 / span 2; + grid-row: 2; + max-height:2.4em; + overflow:hidden; + } + + .medium-item .description{ + grid-column: 2 / span 2; + grid-row: 4; + } + .medium-item .badges{ + grid-column: 2 / span 2; + grid-row: 5; + } + /* thumbnail size */ + .medium-item img{ + /*height:138px; + width:246px;*/ + height:100%; + justify-self:center; + } + +.small-item-box{ + color: #767676; + font-size: 12px; + + display:grid; + grid-template-columns: 1fr 30px; + grid-template-rows: 94px; +} + +.small-item{ + background-color:#bcbcbc; + align-content: start; + text-decoration:none; + + display: grid; + grid-template-columns: 168px 1fr; + grid-column-gap: 5px; + grid-template-rows: auto auto auto 1fr; +} + .small-item .title{ + grid-column:2; + grid-row:1; + margin:0; + + color: #333; + font-size: 16px; + font-weight: 500; + text-decoration:initial; + min-width: 0; + justify-self:start; + + overflow:hidden; + max-height: 3.3em; + line-height: 1.1em; + } + .small-item address{ + grid-column: 2; + grid-row: 2; + justify-self: start; + } + + .small-item .views{ + grid-column: 2; + grid-row: 3; + justify-self:start; + } + /* thumbnail size */ + .small-item img{ + /*height:94px; + width:168px;*/ + height:100%; + justify-self:center; + } + +.item-checkbox{ + justify-self:start; + align-self:center; + height:30px; + width:30px; + + grid-column: 2; +} + +/* ---Thumbnails for videos---- */ +.video-thumbnail-box{ + max-height:100%; + + grid-column:1; + grid-row:1 / span 6; + + display:grid; + grid-template-columns: 1fr 0fr; +} + .video-thumbnail-img{ + grid-column:1 / span 2; + grid-row:1; + } + .video-duration{ + grid-column: 2; + grid-row: 1; + align-self: end; + opacity: .8; + color: #ffffff; + font-size: 12px; + background-color: #000000; + } + +/* ---Thumbnails for playlists---- */ +.playlist-thumbnail-box{ + max-height:100%; + + grid-column:1; + grid-row:1 / span 6; + + display:grid; + grid-template-columns: 3fr 2fr; +} + .playlist-thumbnail-img{ + grid-column:1 / span 2; + grid-row:1; + } + .playlist-thumbnail-info{ + grid-column:2; + grid-row:1; + + display: grid; + align-items:center; + + text-align:center; + white-space: pre-line; + opacity: .8; + color: #cfcfcf; + background-color: #000000; + } + +.page-button-row{ + justify-self:center; + display: grid; + grid-auto-columns: 40px; + grid-auto-flow: column; + height: 40px; +} + .page-button{ + background-color: #e9e9e9; + border-style: outset; + border-width: 2px; + font-weight: bold; + text-align: center; + } +.sort-button{ + background-color: #d0d0d0; + padding: 2px; + justify-self: start; +} diff --git a/youtube/templates/error.html b/youtube/templates/error.html new file mode 100644 index 0000000..f253807 --- /dev/null +++ b/youtube/templates/error.html @@ -0,0 +1,17 @@ + + + + + Error + + + + + + +{{ header|safe }} +
+{{ error_message }} +
+ + diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html new file mode 100644 index 0000000..7e83306 --- /dev/null +++ b/youtube/templates/watch.html @@ -0,0 +1,222 @@ + + + + + {{ title }} + + + + + + + +{{ header|safe }} +
+
+
+
+ + + +

{{ title }}

+{% if unlisted %} + Unlisted +{% endif %} +
Uploaded by {{ uploader }}
+ {{ views }} views + + + + +
+ +
+{% for format in download_formats %} + + {{ format['ext'] }} + {{ format['resolution'] }} + {{ format['note'] }} + +{% endfor %} +
+
+ + + {{ description }} +
+{{ music_list|safe }} +
+{{ comments|safe }} +
+ + + + + + +
+ + + + + + + diff --git a/youtube/watch.py b/youtube/watch.py index 06b525a..48fd7e3 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -1,127 +1,17 @@ +from youtube import yt_app from youtube import util, html_common, comments +import settings + +from flask import request +import flask from youtube_dl.YoutubeDL import YoutubeDL from youtube_dl.extractor.youtube import YoutubeError import json -import urllib -from string import Template import html - import gevent -import settings import os -video_height_priority = (360, 480, 240, 720, 1080) - - -_formats = { - '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'}, - '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'}, - '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'}, - '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'}, - '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'}, - '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}, - '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}, - '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}, - # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well - '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'}, - '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}, - '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'}, - '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'}, - '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'}, - '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'}, - '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'}, - '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}, - '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'}, - - - # 3D videos - '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20}, - '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20}, - '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20}, - '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20}, - '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20}, - '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20}, - '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20}, - - # Apple HTTP Live Streaming - '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}, - '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}, - '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10}, - '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10}, - '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10}, - '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10}, - '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10}, - '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10}, - - # DASH mp4 video - '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559) - '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'}, - '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60}, - '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60}, - '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'}, - - # Dash mp4 audio - '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'}, - '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'}, - '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'}, - '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'}, - '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'}, - '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'}, - '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'}, - - # Dash webm - '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'}, - '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'}, - '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) - '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60}, - '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60}, - '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60}, - '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'}, - '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60}, - - # Dash webm audio - '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128}, - '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256}, - - # Dash webm audio with opus inside - '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50}, - '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70}, - '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160}, - - # RTMP (unnamed) - '_rtmp': {'protocol': 'rtmp'}, -} - - - - - - -with open("yt_watch_template.html", "r") as file: - yt_watch_template = Template(file.read()) - def get_related_items_html(info): result = "" @@ -156,51 +46,45 @@ def watch_page_related_playlist_info(item): 'thumbnail': util.get_thumbnail_url(item['video_id']), } - -def sort_formats(info): - sorted_formats = info['formats'].copy() - sorted_formats.sort(key=lambda x: util.default_multi_get(_formats, x['format_id'], 'height', default=0)) - for index, format in enumerate(sorted_formats): - if util.default_multi_get(_formats, format['format_id'], 'height', default=0) >= 360: - break - sorted_formats = sorted_formats[index:] + sorted_formats[0:index] - sorted_formats = [format for format in info['formats'] if format['acodec'] != 'none' and format['vcodec'] != 'none'] - return sorted_formats - -source_tag_template = Template(''' -''') -def formats_html(formats): - result = '' - for format in formats: - result += source_tag_template.substitute( - src=format['url'], - type='audio/' + format['ext'] if format['vcodec'] == "none" else 'video/' + format['ext'], - ) - return result +def get_video_sources(info): + video_sources = [] + for format in info['formats']: + if format['acodec'] != 'none' and format['vcodec'] != 'none': + video_sources.append({ + 'src': format['url'], + 'type': 'video/' + format['ext'], + }) + return video_sources -subtitles_tag_template = Template(''' -''') -def subtitles_html(info): - result = '' +def get_subtitle_sources(info): + sources = [] default_found = False - default = '' + default = None for language, formats in info['subtitles'].items(): for format in formats: if format['ext'] == 'vtt': - append = subtitles_tag_template.substitute( - src = html.escape('/' + format['url']), - label = html.escape(language), - srclang = html.escape(language), - default = 'default' if language == settings.subtitles_language and settings.subtitles_mode > 0 else '', - ) + source = { + 'url': '/' + format['url'], + 'label': language, + 'srclang': language, + + # set as on by default if this is the preferred language and a default-on subtitles mode is in settings + 'on': language == settings.subtitles_language and settings.subtitles_mode > 0, + } + if language == settings.subtitles_language: default_found = True - default = append + default = source else: - result += append + result.append(source) break - result += default + + # Put it at the end to avoid browser bug when there are too many languages + # (in firefox, it is impossible to select a language near the top of the list because it is cut off) + if default_found: + sources.append(default) + try: formats = info['automatic_captions'][settings.subtitles_language] except KeyError: @@ -208,19 +92,60 @@ def subtitles_html(info): else: for format in formats: if format['ext'] == 'vtt': - result += subtitles_tag_template.substitute( - src = html.escape('/' + format['url']), - label = settings.subtitles_language + ' - Automatic', - srclang = settings.subtitles_language, - default = 'default' if settings.subtitles_mode == 2 and not default_found else '', - ) - return result + sources.append({ + 'url': '/' + format['url'], + 'label': settings.subtitles_language + ' - Automatic', + 'srclang': settings.subtitles_language, + + # set as on by default if this is the preferred language and a default-on subtitles mode is in settings + 'on': settings.subtitles_mode == 2 and not default_found, + + }) + + return sources + + +def get_music_list_html(music_list): + if len(music_list) == 0: + music_list_html = '' + else: + # get the set of attributes which are used by atleast 1 track + # so there isn't an empty, extraneous album column which no tracks use, for example + used_attributes = set() + for track in music_list: + used_attributes = used_attributes | track.keys() + + # now put them in the right order + ordered_attributes = [] + for attribute in ('Artist', 'Title', 'Album'): + if attribute.lower() in used_attributes: + ordered_attributes.append(attribute) + + music_list_html = '''
+ + + +''' + # table headings + for attribute in ordered_attributes: + music_list_html += "\n" + music_list_html += '''\n''' + + for track in music_list: + music_list_html += '''\n''' + for attribute in ordered_attributes: + try: + value = track[attribute.lower()] + except KeyError: + music_list_html += '''''' + else: + music_list_html += '''''' + music_list_html += '''\n''' + music_list_html += '''
Music
" + attribute + "
''' + html.escape(value) + '''
\n''' + return music_list_html -more_comments_template = Template('''More comments''') -download_link_template = Template(''' - $ext $resolution $note''') def extract_info(downloader, *args, **kwargs): try: @@ -228,136 +153,87 @@ def extract_info(downloader, *args, **kwargs): except YoutubeError as e: return str(e) -music_list_table_row = Template(''' - $attribute - $value -''') -def get_watch_page(env, start_response): - video_id = env['parameters']['v'][0] - if len(video_id) < 11: - start_response('404 Not Found', [('Content-type', 'text/plain'),]) - return b'Incomplete video id (too short): ' + video_id.encode('ascii') - - start_response('200 OK', [('Content-type','text/html'),]) - - lc = util.default_multi_get(env['parameters'], 'lc', 0, default='') - if settings.route_tor: - proxy = 'socks5://127.0.0.1:9150/' - else: - proxy = '' - downloader = YoutubeDL(params={'youtube_include_dash_manifest':False, 'proxy':proxy}) - tasks = ( - gevent.spawn(comments.video_comments, video_id, int(settings.default_comment_sorting), lc=lc ), - gevent.spawn(extract_info, downloader, "https://www.youtube.com/watch?v=" + video_id, download=False) - ) - gevent.joinall(tasks) - comments_html, info = tasks[0].value, tasks[1].value - - - #comments_html = comments.comments_html(video_id(url)) - #info = YoutubeDL().extract_info(url, download=False) - - #chosen_format = choose_format(info) - - if isinstance(info, str): # youtube error - return html_common.yt_basic_template.substitute( - page_title = "Error", - style = "", - header = html_common.get_header(), - page = html.escape(info), - ).encode('utf-8') - - sorted_formats = sort_formats(info) - - video_info = { - "duration": util.seconds_to_timestamp(info["duration"]), - "id": info['id'], - "title": info['title'], - "author": info['uploader'], - } - - upload_year = info["upload_date"][0:4] - upload_month = info["upload_date"][4:6] - upload_day = info["upload_date"][6:8] - upload_date = upload_month + "/" + upload_day + "/" + upload_year - - if settings.enable_related_videos: - related_videos_html = get_related_items_html(info) - else: - related_videos_html = '' - music_list = info['music_list'] - if len(music_list) == 0: - music_list_html = '' - else: - # get the set of attributes which are used by atleast 1 track - # so there isn't an empty, extraneous album column which no tracks use, for example - used_attributes = set() - for track in music_list: - used_attributes = used_attributes | track.keys() - - # now put them in the right order - ordered_attributes = [] - for attribute in ('Artist', 'Title', 'Album'): - if attribute.lower() in used_attributes: - ordered_attributes.append(attribute) - - music_list_html = '''
- - - -''' - # table headings - for attribute in ordered_attributes: - music_list_html += "\n" - music_list_html += '''\n''' - for track in music_list: - music_list_html += '''\n''' - for attribute in ordered_attributes: - try: - value = track[attribute.lower()] - except KeyError: - music_list_html += '''''' - else: - music_list_html += '''''' - music_list_html += '''\n''' - music_list_html += '''
Music
" + attribute + "
''' + html.escape(value) + '''
\n''' - if settings.gather_googlevideo_domains: - with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: - url = info['formats'][0]['url'] - subdomain = url[0:url.find(".googlevideo.com")] - f.write(subdomain + "\n") - - download_options = '' - for format in info['formats']: - download_options += download_link_template.substitute( - url = html.escape(format['url']), - ext = html.escape(format['ext']), - resolution = html.escape(downloader.format_resolution(format)), - note = html.escape(downloader._format_note(format)), - ) - - - page = yt_watch_template.substitute( - video_title = html.escape(info["title"]), - page_title = html.escape(info["title"]), - header = html_common.get_header(), - uploader = html.escape(info["uploader"]), - uploader_channel_url = '/' + info["uploader_url"], - upload_date = upload_date, - views = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("view_count", None)), - likes = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("like_count", None)), - dislikes = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("dislike_count", None)), - download_options = download_options, - video_info = html.escape(json.dumps(video_info)), - description = html.escape(info["description"]), - video_sources = formats_html(sorted_formats) + subtitles_html(info), - related = related_videos_html, - - comments = comments_html, - - music_list = music_list_html, - is_unlisted = 'Unlisted' if info['unlisted'] else '', - ) - return page.encode('utf-8') + +@yt_app.route('/watch') +def get_watch_page(): + video_id = request.args['v'] + if len(video_id) < 11: + abort(404) + abort(Response('Incomplete video id (too short): ' + video_id)) + + lc = request.args.get('lc', '') + if settings.route_tor: + proxy = 'socks5://127.0.0.1:9150/' + else: + proxy = '' + yt_dl_downloader = YoutubeDL(params={'youtube_include_dash_manifest':False, 'proxy':proxy}) + tasks = ( + gevent.spawn(comments.video_comments, video_id, int(settings.default_comment_sorting), lc=lc ), + gevent.spawn(extract_info, yt_dl_downloader, "https://www.youtube.com/watch?v=" + video_id, download=False) + ) + gevent.joinall(tasks) + comments_html, info = tasks[0].value, tasks[1].value + + if isinstance(info, str): # youtube error + return flask.render_template('error.html', header = html_common.get_header, error_mesage = info) + + video_info = { + "duration": util.seconds_to_timestamp(info["duration"]), + "id": info['id'], + "title": info['title'], + "author": info['uploader'], + } + + upload_year = info["upload_date"][0:4] + upload_month = info["upload_date"][4:6] + upload_day = info["upload_date"][6:8] + upload_date = upload_month + "/" + upload_day + "/" + upload_year + + if settings.enable_related_videos: + related_videos_html = get_related_items_html(info) + else: + related_videos_html = '' + + + if settings.gather_googlevideo_domains: + with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: + url = info['formats'][0]['url'] + subdomain = url[0:url.find(".googlevideo.com")] + f.write(subdomain + "\n") + + + download_formats = [] + + for format in info['formats']: + download_formats.append({ + 'url': format['url'], + 'ext': format['ext'], + 'resolution': yt_dl_downloader.format_resolution(format), + 'note': yt_dl_downloader._format_note(format), + }) + + + return flask.render_template('watch.html', + header = html_common.get_header(), + uploader_channel_url = '/' + info['uploader_url'], + upload_date = upload_date, + views = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("view_count", None)), + likes = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("like_count", None)), + dislikes = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("dislike_count", None)), + download_formats = download_formats, + video_info = json.dumps(video_info), + video_sources = get_video_sources(info), + subtitle_sources = get_subtitle_sources(info), + + # TODO: refactor these + related = related_videos_html, + comments = comments_html, + music_list = get_music_list_html(info['music_list']), + + title = info['title'], + uploader = info['uploader'], + description = info['description'], + unlisted = info['unlisted'], + ) diff --git a/youtube/youtube.py b/youtube/youtube.py deleted file mode 100644 index a6a216e..0000000 --- a/youtube/youtube.py +++ /dev/null @@ -1,105 +0,0 @@ -import mimetypes -import urllib.parse -import os -from youtube import local_playlist, watch, search, playlist, channel, comments, post_comment, accounts, util -import settings -YOUTUBE_FILES = ( - "/shared.css", - '/comments.css', - '/favicon.ico', -) -get_handlers = { - 'search': search.get_search_page, - '': search.get_search_page, - 'watch': watch.get_watch_page, - 'playlist': playlist.get_playlist_page, - - 'channel': channel.get_channel_page, - 'user': channel.get_channel_page_general_url, - 'c': channel.get_channel_page_general_url, - - 'playlists': local_playlist.get_playlist_page, - - 'comments': comments.get_comments_page, - 'post_comment': post_comment.get_post_comment_page, - 'delete_comment': post_comment.get_delete_comment_page, - 'login': accounts.get_account_login_page, -} -post_handlers = { - 'edit_playlist': local_playlist.edit_playlist, - 'playlists': local_playlist.path_edit_playlist, - - 'login': accounts.add_account, - 'comments': post_comment.post_comment, - 'post_comment': post_comment.post_comment, - 'delete_comment': post_comment.delete_comment, -} - -def youtube(env, start_response): - path, method, query_string = env['PATH_INFO'], env['REQUEST_METHOD'], env['QUERY_STRING'] - env['qs_parameters'] = urllib.parse.parse_qs(query_string) - env['parameters'] = dict(env['qs_parameters']) - - path_parts = path.rstrip('/').lstrip('/').split('/') - env['path_parts'] = path_parts - - if method == "GET": - try: - handler = get_handlers[path_parts[0]] - except KeyError: - pass - else: - return handler(env, start_response) - - if path in YOUTUBE_FILES: - with open("youtube" + path, 'rb') as f: - mime_type = mimetypes.guess_type(path)[0] or 'application/octet-stream' - start_response('200 OK', (('Content-type',mime_type),) ) - return f.read() - - elif path.startswith("/data/playlist_thumbnails/"): - with open(os.path.join(settings.data_dir, os.path.normpath(path[6:])), 'rb') as f: - start_response('200 OK', (('Content-type', "image/jpeg"),) ) - return f.read() - - elif path.startswith("/api/"): - start_response('200 OK', [('Content-type', 'text/vtt'),] ) - result = util.fetch_url('https://www.youtube.com' + path + ('?' + query_string if query_string else '')) - result = result.replace(b"align:start position:0%", b"") - return result - - elif path == "/opensearch.xml": - with open("youtube" + path, 'rb') as f: - mime_type = mimetypes.guess_type(path)[0] or 'application/octet-stream' - start_response('200 OK', (('Content-type',mime_type),) ) - return f.read().replace(b'$port_number', str(settings.port_number).encode()) - - elif path == "/comment_delete_success": - start_response('200 OK', [('Content-type', 'text/plain'),] ) - return b'Successfully deleted comment' - - elif path == "/comment_delete_fail": - start_response('200 OK', [('Content-type', 'text/plain'),] ) - return b'Failed to deleted comment' - - else: - return channel.get_channel_page_general_url(env, start_response) - - elif method == "POST": - post_parameters = urllib.parse.parse_qs(env['wsgi.input'].read().decode()) - env['post_parameters'] = post_parameters - env['parameters'].update(post_parameters) - - try: - handler = post_handlers[path_parts[0]] - except KeyError: - pass - else: - return handler(env, start_response) - - start_response('404 Not Found', [('Content-type', 'text/plain'),]) - return b'404 Not Found' - - else: - start_response('501 Not Implemented', [('Content-type', 'text/plain'),]) - return b'501 Not Implemented' -- cgit v1.2.3 From 1ba241186299df50a94efd3d410a4422bdc2d6c3 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 17 Jun 2019 20:47:58 -0700 Subject: Inherit from base template --- youtube/html_common.py | 109 ------------------------------------------ youtube/templates/base.html | 111 +++++++++++++++++++++++++++++++++++++++++++ youtube/templates/error.html | 25 ++++------ youtube/templates/watch.html | 30 +++--------- youtube/watch.py | 5 +- 5 files changed, 128 insertions(+), 152 deletions(-) create mode 100644 youtube/templates/base.html (limited to 'youtube') diff --git a/youtube/html_common.py b/youtube/html_common.py index 8e65a1f..b8ea0d6 100644 --- a/youtube/html_common.py +++ b/youtube/html_common.py @@ -104,115 +104,6 @@ medium_channel_item_template = Template(''' - -header_template = Template(''' -
- - - -
-
- - -$playlists - - - -
- Local playlists -
-
-''') -playlist_option_template = Template('''''') -def get_header(search_box_value=""): - playlists = '' - for name in local_playlist.get_playlist_names(): - playlists += playlist_option_template.substitute(name = name) - return header_template.substitute(playlists = playlists, search_box_value = html.escape(search_box_value)) - - - - - - - - - - - def badges_html(badges): return ' | '.join(map(html.escape, badges)) diff --git a/youtube/templates/base.html b/youtube/templates/base.html new file mode 100644 index 0000000..e98f972 --- /dev/null +++ b/youtube/templates/base.html @@ -0,0 +1,111 @@ + + + + + {% block page_title %}{% endblock %} + + + + + + + +
+ + +
+
+ + + {% for playlist_name in header_playlist_names %} + + {% endfor %} + + + +
+ Local playlists +
+
+
+{% block main %} +{% endblock %} +
+ + diff --git a/youtube/templates/error.html b/youtube/templates/error.html index f253807..1f33c44 100644 --- a/youtube/templates/error.html +++ b/youtube/templates/error.html @@ -1,17 +1,8 @@ - - - - - Error - - - - - - -{{ header|safe }} -
-{{ error_message }} -
- - +{% extends "base.html" %} + +{% block page_title %}Error{% endblock %} + +{% block main %} + {{ error_message }} +{% endblock %} + diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html index 7e83306..f00413d 100644 --- a/youtube/templates/watch.html +++ b/youtube/templates/watch.html @@ -1,13 +1,6 @@ - - - - - {{ title }} - - - - - - - -{{ header|safe }} -
+{% endblock style %} + +{% block main %}
@@ -212,11 +203,4 @@ {{ related|safe }} -
- - - - - - - +{% endblock main %} diff --git a/youtube/watch.py b/youtube/watch.py index 48fd7e3..72268a7 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -1,5 +1,5 @@ from youtube import yt_app -from youtube import util, html_common, comments +from youtube import util, html_common, comments, local_playlist import settings from flask import request @@ -214,9 +214,8 @@ def get_watch_page(): 'note': yt_dl_downloader._format_note(format), }) - return flask.render_template('watch.html', - header = html_common.get_header(), + header_playlist_names = local_playlist.get_playlist_names(), uploader_channel_url = '/' + info['uploader_url'], upload_date = upload_date, views = (lambda x: '{:,}'.format(x) if x is not None else "")(info.get("view_count", None)), -- cgit v1.2.3 From e33bae2e50799e3617432f294cf36581404a5114 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 17 Jun 2019 20:57:10 -0700 Subject: watch: Fix error page --- youtube/watch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'youtube') diff --git a/youtube/watch.py b/youtube/watch.py index 72268a7..ac70fd4 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -160,8 +160,8 @@ def extract_info(downloader, *args, **kwargs): def get_watch_page(): video_id = request.args['v'] if len(video_id) < 11: - abort(404) - abort(Response('Incomplete video id (too short): ' + video_id)) + flask.abort(404) + flask.abort(flask.Response('Incomplete video id (too short): ' + video_id)) lc = request.args.get('lc', '') if settings.route_tor: -- cgit v1.2.3 From 1156b0998758ee803c7e8ae0cc2beb5181c232a3 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 21:41:41 -0700 Subject: Refactor search page --- youtube/search.py | 119 +++++++++++--------------- youtube/static/shared.css | 6 ++ youtube/templates/base.html | 4 +- youtube/templates/common_elements.html | 152 +++++++++++++++++++++++++++++++++ youtube/templates/search.html | 54 ++++++++++++ youtube/yt_data_extract.py | 113 ++++++++++++++++++++---- 6 files changed, 363 insertions(+), 85 deletions(-) create mode 100644 youtube/templates/common_elements.html create mode 100644 youtube/templates/search.html (limited to 'youtube') diff --git a/youtube/search.py b/youtube/search.py index 0cef0f3..fcc352f 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -1,16 +1,12 @@ -from youtube import util, html_common, yt_data_extract, proto +from youtube import util, yt_data_extract, proto, local_playlist +from youtube import yt_app import json import urllib -import html -from string import Template import base64 from math import ceil - - -with open("yt_search_results_template.html", "r") as file: - yt_search_results_template = file.read() - +from flask import request +import flask # Sort: 1 # Upload date: 2 @@ -58,41 +54,32 @@ def get_search_json(query, page, autocorrect, sort, filters): content = util.fetch_url(url, headers=headers, report_text="Got search results") info = json.loads(content) return info - -showing_results_for = Template(''' -
Showing results for $corrected_query
-
Search instead for $original_query
-''') -did_you_mean = Template(''' -
Did you mean $corrected_query
-''') -def get_search_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - parameters = env['parameters'] - if len(parameters) == 0: - return html_common.yt_basic_template.substitute( - page_title = "Search", - header = html_common.get_header(), - style = '', - page = '', - ).encode('utf-8') - query = parameters["query"][0] - page = parameters.get("page", "1")[0] - autocorrect = int(parameters.get("autocorrect", "1")[0]) - sort = int(parameters.get("sort", "0")[0]) + +@yt_app.route('/search') +def get_search_page(): + if len(request.args) == 0: + return flask.render_template('base.html', title="Search") + + if 'query' not in request.args: + abort(400) + + query = request.args.get("query") + page = request.args.get("page", "1") + autocorrect = int(request.args.get("autocorrect", "1")) + sort = int(request.args.get("sort", "0")) filters = {} - filters['time'] = int(parameters.get("time", "0")[0]) - filters['type'] = int(parameters.get("type", "0")[0]) - filters['duration'] = int(parameters.get("duration", "0")[0]) + filters['time'] = int(request.args.get("time", "0")) + filters['type'] = int(request.args.get("type", "0")) + filters['duration'] = int(request.args.get("duration", "0")) info = get_search_json(query, page, autocorrect, sort, filters) estimated_results = int(info[1]['response']['estimatedResults']) estimated_pages = ceil(estimated_results/20) results = info[1]['response']['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'] - - corrections = '' - result_list_html = "" + + parsed_results = [] + corrections = {'type': None} for renderer in results: type = list(renderer.keys())[0] if type == 'shelfRenderer': @@ -102,41 +89,39 @@ def get_search_page(env, start_response): corrected_query_string = parameters.copy() corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']] corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True) - corrections = did_you_mean.substitute( - corrected_query_url = corrected_query_url, - corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), - ) + + corrections = { + 'type': 'did_you_mean', + 'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), + 'corrected_query_url': corrected_query_url, + } continue if type == 'showingResultsForRenderer': renderer = renderer[type] no_autocorrect_query_string = parameters.copy() no_autocorrect_query_string['autocorrect'] = ['0'] no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True) - corrections = showing_results_for.substitute( - corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), - original_query_url = no_autocorrect_query_url, - original_query = html.escape(renderer['originalQuery']['simpleText']), - ) + + corrections = { + 'type': 'showing_results_for', + 'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), + 'original_query_url': no_autocorrect_query_url, + 'original_query': renderer['originalQuery']['simpleText'], + } continue - result_list_html += html_common.renderer_html(renderer, current_query_string=env['QUERY_STRING']) - - page = int(page) - if page <= 5: - page_start = 1 - page_end = min(9, estimated_pages) - else: - page_start = page - 4 - page_end = min(page + 4, estimated_pages) - - - result = Template(yt_search_results_template).substitute( - header = html_common.get_header(query), - results = result_list_html, - page_title = query + " - Search", - search_box_value = html.escape(query), - number_of_results = '{:,}'.format(estimated_results), - number_of_pages = '{:,}'.format(estimated_pages), - page_buttons = html_common.page_buttons_html(page, estimated_pages, util.URL_ORIGIN + "/search", env['QUERY_STRING']), - corrections = corrections - ) - return result.encode('utf-8') + + info = yt_data_extract.parse_info_prepare_for_html(renderer) + if info['type'] != 'unsupported': + parsed_results.append(info) + + return flask.render_template('search.html', + header_playlist_names = local_playlist.get_playlist_names(), + query = query, + estimated_results = estimated_results, + estimated_pages = estimated_pages, + corrections = corrections, + results = parsed_results, + parameters_dictionary = request.args, + ) + + diff --git a/youtube/static/shared.css b/youtube/static/shared.css index 1b25d7f..a360972 100644 --- a/youtube/static/shared.css +++ b/youtube/static/shared.css @@ -219,6 +219,12 @@ address{ max-height:2.4em; overflow:hidden; } + .medium-item .stats > *::after{ + content: " | "; + } + .medium-item .stats > *:last-child::after{ + content: ""; + } .medium-item .description{ grid-column: 2 / span 2; diff --git a/youtube/templates/base.html b/youtube/templates/base.html index e98f972..eafd369 100644 --- a/youtube/templates/base.html +++ b/youtube/templates/base.html @@ -2,13 +2,14 @@ - {% block page_title %}{% endblock %} + {% block page_title %}{{ title }}{% endblock %} @@ -105,6 +106,7 @@
{% block main %} +{{ main }} {% endblock %}
diff --git a/youtube/templates/common_elements.html b/youtube/templates/common_elements.html new file mode 100644 index 0000000..9f2aa3f --- /dev/null +++ b/youtube/templates/common_elements.html @@ -0,0 +1,152 @@ +{% macro text_runs(runs) %} + {%- if runs[0] is mapping -%} + {%- for text_run in runs -%} + {%- if text_run.get("bold", false) -%} + {{ text_run["text"] }} + {%- elif text_run.get('italics', false) -%} + {{ text_run["text"] }} + {%- else -%} + {{ text_run["text"] }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ runs }} + {%- endif -%} +{% endmacro %} + +{% macro small_item(info) %} +
+
+ {% if info['type'] == 'video' %} + + + {{ info['duration'] }} + + {{ info['title'] }} + +
{{ info['author'] }}
+ {{ info['views'] }} + + {% elif info['type'] == 'playlist' %} + + +
+ {{ info['size'] }} +
+ + {{ info['title'] }} + +
{{ info['author'] }}
+ {% else %} + Error: unsupported item type + {% endif %} +
+ {% if info['type'] == 'video' %} + + {% endif %} +
+{% endmacro %} + +{% macro get_stats(info) %} + {% if 'author_url' is in(info) %} +
By {{ info['author'] }}
+ {% else %} +
{{ info['author'] }}
+ {% endif %} + {% if 'views' is in(info) %} + {{ info['views'] }} + {% endif %} + {% if 'published' is in(info) %} + + {% endif %} +{% endmacro %} + + + +{% macro medium_item(info) %} +
+
+ {% if info['type'] == 'video' %} + + + {{ info['duration'] }} + + + {{ info['title'] }} + +
+ {{ get_stats(info) }} +
+ + {{ text_runs(info['description']) }} + {{ info['badges']|join(' | ') }} + {% elif info['type'] == 'playlist' %} + + +
+ {{ info['size'] }} +
+ + + {{ info['title'] }} + +
+ {{ get_stats(info) }} +
+ {% elif info['type'] == 'channel' %} + + + + + {{ info['title'] }} + + {{ info['subscriber_count'] }} + {{ info['size'] }} + + {{ text_runs(info['description']) }} + {% else %} + Error: unsupported item type + {% endif %} +
+ {% if info['type'] == 'video' %} + + {% endif %} +
+{% endmacro %} + + +{% macro item(info) %} + {% if info['item_size'] == 'small' %} + {{ small_item(info) }} + {% elif info['item_size'] == 'medium' %} + {{ medium_item(info) }} + {% else %} + Error: Unknown item size + {% endif %} +{% endmacro %} + + + +{% macro page_buttons(estimated_pages, url, parameters_dictionary) %} + {% set current_page = parameters_dictionary.get('page', 1)|int %} + {% set parameters_dictionary = parameters_dictionary.to_dict() %} + {% if current_page is le(5) %} + {% set page_start = 1 %} + {% set page_end = [9, estimated_pages]|min %} + {% else %} + {% set page_start = current_page - 4 %} + {% set page_end = [current_page + 4, estimated_pages]|min %} + {% endif %} + + {% for page in range(page_start, page_end+1) %} + {% if page == current_page %} +
{{ page }}
+ {% else %} + {# IMPORTANT: Jinja SUCKS #} + {# https://stackoverflow.com/questions/36886650/how-to-add-a-new-entry-into-a-dictionary-object-while-using-jinja2 #} + {% set _ = parameters_dictionary.__setitem__('page', page) %} + {{ page }} + {% endif %} + {% endfor %} + +{% endmacro %} diff --git a/youtube/templates/search.html b/youtube/templates/search.html new file mode 100644 index 0000000..1086cfd --- /dev/null +++ b/youtube/templates/search.html @@ -0,0 +1,54 @@ +{% set search_box_value = query %} +{% extends "base.html" %} +{% block page_title %}{{ query + ' - Search' }}{% endblock %} +{% import "common_elements.html" as common_elements %} +{% block style %} + main{ + display:grid; + grid-template-columns: minmax(0px, 1fr) 800px minmax(0px,2fr); + max-width:100vw; + } + + + #number-of-results{ + font-weight:bold; + } + #result-info{ + grid-row: 1; + grid-column:2; + align-self:center; + } + .page-button-row{ + grid-column: 2; + justify-self: center; + } + + + .item-list{ + grid-row: 2; + grid-column: 2; + } + .badge{ + background-color:#cccccc; + } +{% endblock style %} + +{% block main %} +
+
Approximately {{ '{:,}'.format(estimated_results) }} results ({{ '{:,}'.format(estimated_pages) }} pages)
+{% if corrections['type'] == 'showing_results_for' %} + + +{% elif corrections['type'] == 'did_you_mean' %} + +{% endif %} +
+
+ {% for info in results %} + {{ common_elements.item(info) }} + {% endfor %} +
+ +{% endblock main %} diff --git a/youtube/yt_data_extract.py b/youtube/yt_data_extract.py index 5483911..a487c57 100644 --- a/youtube/yt_data_extract.py +++ b/youtube/yt_data_extract.py @@ -1,4 +1,7 @@ +from youtube import util + import html +import json # videos (all of type str): @@ -138,9 +141,83 @@ dispatch = { } -def renderer_info(renderer): +def ajax_info(item_json): + try: + info = {} + for key, node in item_json.items(): + try: + simple_key, function = dispatch[key] + except KeyError: + continue + info[simple_key] = function(node) + return info + except KeyError: + print(item_json) + raise + + + +def prefix_urls(item): + try: + item['thumbnail'] = '/' + item['thumbnail'].lstrip('/') + except KeyError: + pass + + try: + item['author_url'] = util.URL_ORIGIN + item['author_url'] + except KeyError: + pass + +def add_extra_html_info(item): + if item['type'] == 'video': + item['url'] = util.URL_ORIGIN + '/watch?v=' + item['id'] + + video_info = {} + for key in ('id', 'title', 'author', 'duration'): + try: + video_info[key] = item[key] + except KeyError: + video_info[key] = '' + + item['video_info'] = json.dumps(video_info) + + elif item['type'] == 'playlist': + item['url'] = util.URL_ORIGIN + '/playlist?list=' + item['id'] + elif item['type'] == 'channel': + item['url'] = util.URL_ORIGIN + "/channel/" + item['id'] + + +def renderer_info(renderer, additional_info={}): + type = list(renderer.keys())[0] + renderer = renderer[type] + info = {} + if type == 'itemSectionRenderer': + return renderer_info(renderer['contents'][0], additional_info) + + if type in ('movieRenderer', 'clarificationRenderer'): + info['type'] = 'unsupported' + return info + + info.update(additional_info) + + if type.startswith('compact'): + info['item_size'] = 'small' + else: + info['item_size'] = 'medium' + + if type in ('compactVideoRenderer', 'videoRenderer', 'gridVideoRenderer'): + info['type'] = 'video' + elif type in ('playlistRenderer', 'compactPlaylistRenderer', 'gridPlaylistRenderer', + 'radioRenderer', 'compactRadioRenderer', 'gridRadioRenderer', + 'showRenderer', 'compactShowRenderer', 'gridShowRenderer'): + info['type'] = 'playlist' + elif type == 'channelRenderer': + info['type'] = 'channel' + else: + info['type'] = 'unsupported' + return info + try: - info = {} if 'viewCountText' in renderer: # prefer this one as it contains all the digits info['views'] = get_text(renderer['viewCountText']) elif 'shortViewCountText' in renderer: @@ -183,23 +260,25 @@ def renderer_info(renderer): except KeyError: continue info[simple_key] = function(node) + if info['type'] == 'video' and 'duration' not in info: + info['duration'] = 'Live' + return info except KeyError: print(renderer) raise - -def ajax_info(item_json): - try: - info = {} - for key, node in item_json.items(): - try: - simple_key, function = dispatch[key] - except KeyError: - continue - info[simple_key] = function(node) - return info - except KeyError: - print(item_json) - raise - + + + + #print(renderer) + #raise NotImplementedError('Unknown renderer type: ' + type) + return '' + +def parse_info_prepare_for_html(renderer): + item = renderer_info(renderer) + prefix_urls(item) + add_extra_html_info(item) + + return item + -- cgit v1.2.3 From 02962df0526cf265965c442bf9a261ceba55864e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 21:59:33 -0700 Subject: Refactor watch page related videos to use item rendering macro --- youtube/templates/watch.html | 5 ++++- youtube/watch.py | 27 ++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) (limited to 'youtube') diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html index f00413d..122958c 100644 --- a/youtube/templates/watch.html +++ b/youtube/templates/watch.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% import "common_elements.html" as common_elements %} {% block page_title %}{{ title }}{% endblock %} {% block style %} main{ @@ -200,7 +201,9 @@ {% endblock main %} diff --git a/youtube/watch.py b/youtube/watch.py index ac70fd4..818abf2 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -1,5 +1,5 @@ from youtube import yt_app -from youtube import util, html_common, comments, local_playlist +from youtube import util, html_common, comments, local_playlist, yt_data_extract import settings from flask import request @@ -13,16 +13,17 @@ import gevent import os -def get_related_items_html(info): - result = "" +def get_related_items(info): + results = [] for item in info['related_vids']: if 'list' in item: # playlist: - item = watch_page_related_playlist_info(item) - result += html_common.playlist_item_html(item, html_common.small_playlist_item_template) + result = watch_page_related_playlist_info(item) else: - item = watch_page_related_video_info(item) - result += html_common.video_item_html(item, html_common.small_video_item_template) - return result + result = watch_page_related_video_info(item) + yt_data_extract.prefix_urls(result) + yt_data_extract.add_extra_html_info(result) + results.append(result) + return results # json of related items retrieved directly from the watch page has different names for everything @@ -35,6 +36,8 @@ def watch_page_related_video_info(item): except KeyError: result['views'] = '' result['thumbnail'] = util.get_thumbnail_url(item['id']) + result['item_size'] = 'small' + result['type'] = 'video' return result def watch_page_related_playlist_info(item): @@ -44,6 +47,8 @@ def watch_page_related_playlist_info(item): 'id': item['list'], 'first_video_id': item['video_id'], 'thumbnail': util.get_thumbnail_url(item['video_id']), + 'item_size': 'small', + 'type': 'playlist', } def get_video_sources(info): @@ -192,9 +197,9 @@ def get_watch_page(): upload_date = upload_month + "/" + upload_day + "/" + upload_year if settings.enable_related_videos: - related_videos_html = get_related_items_html(info) + related_videos = get_related_items(info) else: - related_videos_html = '' + related_videos = [] if settings.gather_googlevideo_domains: @@ -225,9 +230,9 @@ def get_watch_page(): video_info = json.dumps(video_info), video_sources = get_video_sources(info), subtitle_sources = get_subtitle_sources(info), + related = related_videos, # TODO: refactor these - related = related_videos_html, comments = comments_html, music_list = get_music_list_html(info['music_list']), -- cgit v1.2.3 From 05de30454852c6f55f33c8e8c0ab948bf2c590c2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 21:59:50 -0700 Subject: Fix get_subtitle_sources function --- youtube/watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/watch.py b/youtube/watch.py index 818abf2..1f7a352 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -82,7 +82,7 @@ def get_subtitle_sources(info): default_found = True default = source else: - result.append(source) + sources.append(source) break # Put it at the end to avoid browser bug when there are too many languages -- cgit v1.2.3 From 1c724f4f28804f3f8e41d222576e6fc5d7e68f75 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 22:29:25 -0700 Subject: Search: fix corrections not working --- youtube/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'youtube') diff --git a/youtube/search.py b/youtube/search.py index fcc352f..76a814c 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -86,7 +86,7 @@ def get_search_page(): continue if type == 'didYouMeanRenderer': renderer = renderer[type] - corrected_query_string = parameters.copy() + corrected_query_string = request.args.to_dict(flat=False) corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']] corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True) @@ -98,7 +98,7 @@ def get_search_page(): continue if type == 'showingResultsForRenderer': renderer = renderer[type] - no_autocorrect_query_string = parameters.copy() + no_autocorrect_query_string = request.args.to_dict(flat=False) no_autocorrect_query_string['autocorrect'] = ['0'] no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True) -- cgit v1.2.3 From d105d4520ff0bf529cfb18c9c16a22900ab7f481 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 4 Jul 2019 18:08:14 -0700 Subject: Convert playlist page to flask framework --- youtube/playlist.py | 58 ++++++++++------------ youtube/templates/playlist.html | 106 ++++++++++++++++++++++++++++++++++++++++ youtube/yt_data_extract.py | 6 ++- 3 files changed, 136 insertions(+), 34 deletions(-) create mode 100644 youtube/templates/playlist.html (limited to 'youtube') diff --git a/youtube/playlist.py b/youtube/playlist.py index fbe6448..18ddf49 100644 --- a/youtube/playlist.py +++ b/youtube/playlist.py @@ -1,4 +1,5 @@ -from youtube import util, yt_data_extract, html_common, template, proto +from youtube import util, yt_data_extract, proto +from youtube import yt_app import base64 import urllib @@ -6,10 +7,8 @@ import json import string import gevent import math - -with open("yt_playlist_template.html", "r") as file: - yt_playlist_template = template.Template(file.read()) - +from flask import request +import flask @@ -76,14 +75,15 @@ def get_videos(playlist_id, page): return info -playlist_stat_template = string.Template(''' -
$stat
''') -def get_playlist_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - parameters = env['parameters'] - playlist_id = parameters['list'][0] - page = parameters.get("page", "1")[0] - if page == "1": +@yt_app.route('/playlist') +def get_playlist_page(): + if 'list' not in request.args: + abort(400) + + playlist_id = request.args.get('list') + page = request.args.get('page', '1') + + if page == '1': first_page_json = playlist_first_page(playlist_id) this_page_json = first_page_json else: @@ -98,26 +98,20 @@ def get_playlist_page(env, start_response): video_list = this_page_json['response']['contents']['singleColumnBrowseResultsRenderer']['tabs'][0]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['playlistVideoListRenderer']['contents'] except KeyError: # other pages video_list = this_page_json['response']['continuationContents']['playlistVideoListContinuation']['contents'] - videos_html = '' - for video_json in video_list: - info = yt_data_extract.renderer_info(video_json['playlistVideoRenderer']) - videos_html += html_common.video_item_html(info, html_common.small_video_item_template) + parsed_video_list = [yt_data_extract.parse_info_prepare_for_html(video_json) for video_json in video_list] + + + metadata = yt_data_extract.renderer_info(first_page_json['response']['header']) + yt_data_extract.prefix_urls(metadata) - metadata = yt_data_extract.renderer_info(first_page_json['response']['header']['playlistHeaderRenderer']) video_count = int(metadata['size'].replace(',', '')) - page_buttons = html_common.page_buttons_html(int(page), math.ceil(video_count/20), util.URL_ORIGIN + "/playlist", env['QUERY_STRING']) - - html_ready = html_common.get_html_ready(metadata) - html_ready['page_title'] = html_ready['title'] + ' - Page ' + str(page) - - stats = '' - stats += playlist_stat_template.substitute(stat=html_ready['size'] + ' videos') - stats += playlist_stat_template.substitute(stat=html_ready['views']) - return yt_playlist_template.substitute( - header = html_common.get_header(), - videos = videos_html, - page_buttons = page_buttons, - stats = stats, - **html_ready + metadata['size'] += ' videos' + + return flask.render_template('playlist.html', + video_list = parsed_video_list, + num_pages = math.ceil(video_count/20), + parameters_dictionary = request.args, + + **metadata ).encode('utf-8') diff --git a/youtube/templates/playlist.html b/youtube/templates/playlist.html new file mode 100644 index 0000000..09e382b --- /dev/null +++ b/youtube/templates/playlist.html @@ -0,0 +1,106 @@ +{% extends "base.html" %} +{% block page_title %}{{ title + ' - Page ' + parameters_dictionary.get('page', '1') }}{% endblock %} +{% import "common_elements.html" as common_elements %} +{% block style %} + main{ + display:grid; + grid-template-columns: 3fr 1fr; + } + + + + #left{ + grid-column: 1; + grid-row: 1; + + display: grid; + grid-template-columns: 1fr 800px; + grid-template-rows: 0fr 1fr 0fr; + } + .playlist-metadata{ + grid-column:2; + grid-row:1; + + display:grid; + grid-template-columns: 0fr 1fr; + } + .playlist-thumbnail{ + grid-row: 1 / span 5; + grid-column:1; + justify-self:start; + width:250px; + margin-right: 10px; + } + .playlist-title{ + grid-row: 1; + grid-column:2; + } + .playlist-author{ + grid-row:2; + grid-column:2; + } + .playlist-stats{ + grid-row:3; + grid-column:2; + } + + .playlist-description{ + grid-row:4; + grid-column:2; + min-width:0px; + white-space: pre-line; + } + .page-button-row{ + grid-row: 3; + grid-column: 2; + justify-self: center; + } + + + #right{ + grid-column: 2; + grid-row: 1; + + } + #results{ + + grid-row: 2; + grid-column: 2; + margin-top:10px; + + display: grid; + grid-auto-rows: 0fr; + grid-row-gap: 10px; + + } +{% endblock style %} + +{% block main %} +
+
+ +

{{ title }}

+ {{ author }} +
+
{{ views }}
+
{{ size }}
+
+
{{ description }}
+
+ +
+ {% for info in video_list %} + {{ common_elements.item(info) }} + {% endfor %} +
+ +
+{% endblock main %} + + + + + + diff --git a/youtube/yt_data_extract.py b/youtube/yt_data_extract.py index a487c57..a42b6a2 100644 --- a/youtube/yt_data_extract.py +++ b/youtube/yt_data_extract.py @@ -200,12 +200,12 @@ def renderer_info(renderer, additional_info={}): info.update(additional_info) - if type.startswith('compact'): + if type.startswith('compact') or type.startswith('playlist') or type.startswith('grid'): info['item_size'] = 'small' else: info['item_size'] = 'medium' - if type in ('compactVideoRenderer', 'videoRenderer', 'gridVideoRenderer'): + if type in ('compactVideoRenderer', 'videoRenderer', 'playlistVideoRenderer', 'gridVideoRenderer'): info['type'] = 'video' elif type in ('playlistRenderer', 'compactPlaylistRenderer', 'gridPlaylistRenderer', 'radioRenderer', 'compactRadioRenderer', 'gridRadioRenderer', @@ -213,6 +213,8 @@ def renderer_info(renderer, additional_info={}): info['type'] = 'playlist' elif type == 'channelRenderer': info['type'] = 'channel' + elif type == 'playlistHeaderRenderer': + info['type'] = 'playlist_metadata' else: info['type'] = 'unsupported' return info -- cgit v1.2.3 From 315039bebd8d47820e3c0baeb7ac258db6741582 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 4 Jul 2019 18:20:51 -0700 Subject: common_elements.html: accept missing description as blank --- youtube/templates/common_elements.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/templates/common_elements.html b/youtube/templates/common_elements.html index 9f2aa3f..b140332 100644 --- a/youtube/templates/common_elements.html +++ b/youtube/templates/common_elements.html @@ -78,7 +78,7 @@ {{ get_stats(info) }} - {{ text_runs(info['description']) }} + {{ text_runs(info.get('description', '')) }} {{ info['badges']|join(' | ') }} {% elif info['type'] == 'playlist' %} -- cgit v1.2.3 From 64434b02ca50c2a6324caa1355559bd881ba687e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 6 Jul 2019 18:36:09 -0700 Subject: Convert channel page to flask framework --- youtube/__init__.py | 3 +- youtube/channel.py | 474 +++++++++++++-------------------- youtube/templates/channel.html | 144 ++++++++++ youtube/templates/common_elements.html | 26 +- youtube/yt_data_extract.py | 30 +-- 5 files changed, 363 insertions(+), 314 deletions(-) create mode 100644 youtube/templates/channel.html (limited to 'youtube') diff --git a/youtube/__init__.py b/youtube/__init__.py index 0df56d1..faab170 100644 --- a/youtube/__init__.py +++ b/youtube/__init__.py @@ -1,2 +1,3 @@ import flask -yt_app = flask.Flask(__name__) \ No newline at end of file +yt_app = flask.Flask(__name__) +yt_app.url_map.strict_slashes = False diff --git a/youtube/channel.py b/youtube/channel.py index e9f315b..9cb1e78 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -1,5 +1,6 @@ import base64 from youtube import util, yt_data_extract, html_common +from youtube import yt_app import http_errors import urllib @@ -12,11 +13,8 @@ import gevent import re import functools -with open("yt_channel_items_template.html", "r") as file: - yt_channel_items_template = Template(file.read()) - -with open("yt_channel_about_template.html", "r") as file: - yt_channel_about_template = Template(file.read()) +import flask +from flask import request '''continuation = Proto( Field('optional', 'continuation', 80226972, Proto( @@ -96,11 +94,7 @@ def get_channel_tab(channel_id, page="1", sort=3, tab='videos', view=1): '''with open('debug/channel_debug', 'wb') as f: f.write(content)''' - info = json.loads(content) - return info - - - + return content def get_number_of_videos(channel_id): # Uploads playlist @@ -136,71 +130,22 @@ def get_channel_id(username): response = util.fetch_url(url, util.mobile_ua + headers_1).decode('utf-8') return re.search(r'"channel_id":\s*"([a-zA-Z0-9_-]*)"', response).group(1) -def grid_items_html(items, additional_info={}): - result = ''' ''' - return result - -def list_items_html(items, additional_info={}): - result = ''' ''' - return result - -channel_tab_template = Template('''\n$tab_name''') -channel_search_template = Template(''' -
- - -
''') - -tabs = ('Videos', 'Playlists', 'About') -def channel_tabs_html(channel_id, current_tab, search_box_value=''): - result = '' - for tab_name in tabs: - if tab_name == current_tab: - result += channel_tab_template.substitute( - href_attribute = '', - tab_name = tab_name, - ) - else: - result += channel_tab_template.substitute( - href_attribute = ' href="' + util.URL_ORIGIN + '/channel/' + channel_id + '/' + tab_name.lower() + '"', - tab_name = tab_name, - ) - result += channel_search_template.substitute( - action = util.URL_ORIGIN + "/channel/" + channel_id + "/search", - search_box_value = html.escape(search_box_value), - ) - return result - -channel_sort_button_template = Template('''\n$text''') -sorts = { - "videos": (('1', 'views'), ('2', 'oldest'), ('3', 'newest'),), - "playlists": (('2', 'oldest'), ('3', 'newest'), ('4', 'last video added'),), -} -def channel_sort_buttons_html(channel_id, tab, current_sort): - result = '' - for sort_number, sort_name in sorts[tab]: - if sort_number == str(current_sort): - result += channel_sort_button_template.substitute( - href_attribute='', - text = 'Sorted by ' + sort_name - ) - else: - result += channel_sort_button_template.substitute( - href_attribute=' href="' + util.URL_ORIGIN + '/channel/' + channel_id + '/' + tab + '?sort=' + sort_number + '"', - text = 'Sort by ' + sort_name - ) - return result +def get_channel_search_json(channel_id, query, page): + params = proto.string(2, 'search') + proto.string(15, str(page)) + params = proto.percent_b64encode(params) + ctoken = proto.string(2, channel_id) + proto.string(3, params) + proto.string(11, query) + ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii') + + polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, util.desktop_ua + headers_1) + '''with open('debug/channel_search_debug', 'wb') as f: + f.write(polymer_json)''' + return polymer_json -def get_microformat(response): +def extract_info(polymer_json, tab, html_prepare=True): + response = polymer_json[1]['response'] try: - return response['microformat']['microformatDataRenderer'] + microformat = response['microformat']['microformatDataRenderer'] # channel doesn't exist or was terminated # example terminated channel: https://www.youtube.com/channel/UCnKJeK_r90jDdIuzHXC0Org @@ -209,185 +154,133 @@ def get_microformat(response): result = '' for alert in response['alerts']: result += alert['alertRenderer']['text']['simpleText'] + '\n' - raise http_errors.Code200(result) + flask.abort(200, result) elif 'errors' in response['responseContext']: for error in response['responseContext']['errors']['error']: if error['code'] == 'INVALID_VALUE' and error['location'] == 'browse_id': - raise http_errors.Error404('This channel does not exist') + flask.abort(404, 'This channel does not exist') raise -# example channel with no videos: https://www.youtube.com/user/jungleace -def get_grid_items(response): - try: - return response['continuationContents']['gridContinuation']['items'] - except KeyError: - try: - contents = response['contents'] - except KeyError: - return [] - - item_section = tab_with_content(contents['twoColumnBrowseResultsRenderer']['tabs'])['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0] - try: - return item_section['gridRenderer']['items'] - except KeyError: - if "messageRenderer" in item_section: - return [] - else: - raise + info = {} + info['current_tab'] = tab -def channel_videos_html(polymer_json, current_page=1, current_sort=3, number_of_videos = 1000, current_query_string=''): - response = polymer_json[1]['response'] - microformat = get_microformat(response) + + # stuff from microformat (info given by youtube for every page on channel) + info['description'] = microformat['description'] + info['channel_name'] = microformat['title'] + info['avatar'] = microformat['thumbnail']['thumbnails'][0]['url'] channel_url = microformat['urlCanonical'].rstrip('/') channel_id = channel_url[channel_url.rfind('/')+1:] + info['channel_id'] = channel_id + info['channel_url'] = 'https://www.youtube.com/channel/' + channel_id + + + # empty channel + if 'contents' not in response and 'continuationContents' not in response: + info['items'] = [] + return info + + + # find the tab with content + # example channel where tabs do not have definite index: https://www.youtube.com/channel/UC4gQ8i3FD7YbhOgqUkeQEJg + # TODO: maybe use the 'selected' attribute for this? + if 'continuationContents' not in response: + tab_renderer = None + tab_content = None + for tab_json in response['contents']['twoColumnBrowseResultsRenderer']['tabs']: + try: + tab_renderer = tab_json['tabRenderer'] + except KeyError: + tab_renderer = tab_json['expandableTabRenderer'] + try: + tab_content = tab_renderer['content'] + break + except KeyError: + pass + else: # didn't break + raise Exception("No tabs found with content") + assert tab == tab_renderer['title'].lower() + + + # extract tab-specific info + if tab in ('videos', 'playlists', 'search'): # find the list of items + if 'continuationContents' in response: + try: + items = response['continuationContents']['gridContinuation']['items'] + except KeyError: + items = response['continuationContents']['sectionListContinuation']['contents'] # for search + else: + contents = tab_content['sectionListRenderer']['contents'] + if 'itemSectionRenderer' in contents[0]: + item_section = contents[0]['itemSectionRenderer']['contents'][0] + try: + items = item_section['gridRenderer']['items'] + except KeyError: + if "messageRenderer" in item_section: + items = [] + else: + raise Exception('gridRenderer missing but messageRenderer not found') + else: + items = contents # for search - items = get_grid_items(response) - items_html = grid_items_html(items, {'author': microformat['title']}) - - return yt_channel_items_template.substitute( - header = html_common.get_header(), - channel_title = microformat['title'], - channel_tabs = channel_tabs_html(channel_id, 'Videos'), - sort_buttons = channel_sort_buttons_html(channel_id, 'videos', current_sort), - avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'], - page_title = microformat['title'] + ' - Channel', - items = items_html, - page_buttons = html_common.page_buttons_html(current_page, math.ceil(number_of_videos/30), util.URL_ORIGIN + "/channel/" + channel_id + "/videos", current_query_string), - number_of_results = '{:,}'.format(number_of_videos) + " videos", - ) + # TODO: Fix this URL prefixing shit + additional_info = {'author': info['channel_name'], 'author_url': '/channel/' + channel_id} + if html_prepare: + info['items'] = [yt_data_extract.parse_info_prepare_for_html(renderer, additional_info) for renderer in items] + elif items is not None: + info['items'] = [yt_data_extract.renderer_info(renderer, additional_info) for renderer in items] -def channel_playlists_html(polymer_json, current_sort=3): - response = polymer_json[1]['response'] - microformat = get_microformat(response) - channel_url = microformat['urlCanonical'].rstrip('/') - channel_id = channel_url[channel_url.rfind('/')+1:] + elif tab == 'about': + channel_metadata = tab_content['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['channelAboutFullMetadataRenderer'] - items = get_grid_items(response) - items_html = grid_items_html(items, {'author': microformat['title']}) - - return yt_channel_items_template.substitute( - header = html_common.get_header(), - channel_title = microformat['title'], - channel_tabs = channel_tabs_html(channel_id, 'Playlists'), - sort_buttons = channel_sort_buttons_html(channel_id, 'playlists', current_sort), - avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'], - page_title = microformat['title'] + ' - Channel', - items = items_html, - page_buttons = '', - number_of_results = '', - ) -# Example channel where tabs do not have definite index: https://www.youtube.com/channel/UC4gQ8i3FD7YbhOgqUkeQEJg -def tab_with_content(tabs): - for tab in tabs: - try: - renderer = tab['tabRenderer'] - except KeyError: - renderer = tab['expandableTabRenderer'] - try: - return renderer['content'] - except KeyError: - pass - - raise Exception("No tabs found with content") - -channel_link_template = Template(''' -
  • $text
    • ''') -stat_template = Template(''' -
  • $stat_value
    • ''') -def channel_about_page(polymer_json): - microformat = get_microformat(polymer_json[1]['response']) - avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'] - # my goodness... - channel_metadata = tab_with_content(polymer_json[1]['response']['contents']['twoColumnBrowseResultsRenderer']['tabs'])['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['channelAboutFullMetadataRenderer'] - channel_links = '' - for link_json in channel_metadata.get('primaryLinks', ()): - url = link_json['navigationEndpoint']['urlEndpoint']['url'] - if url.startswith("/redirect"): - query_string = url[url.find('?')+1: ] - url = urllib.parse.parse_qs(query_string)['q'][0] - - channel_links += channel_link_template.substitute( - url = html.escape(url), - text = yt_data_extract.get_plain_text(link_json['title']), - ) + info['links'] = [] + for link_json in channel_metadata.get('primaryLinks', ()): + url = link_json['navigationEndpoint']['urlEndpoint']['url'] + if url.startswith('/redirect'): # youtube puts these on external links to do tracking + query_string = url[url.find('?')+1: ] + url = urllib.parse.parse_qs(query_string)['q'][0] - stats = '' - for stat_name in ('subscriberCountText', 'joinedDateText', 'viewCountText', 'country'): - try: - stat_value = yt_data_extract.get_plain_text(channel_metadata[stat_name]) - except KeyError: - continue - else: - stats += stat_template.substitute(stat_value=stat_value) - try: - description = yt_data_extract.format_text_runs(yt_data_extract.get_formatted_text(channel_metadata['description'])) - except KeyError: - description = '' - return yt_channel_about_template.substitute( - header = html_common.get_header(), - page_title = yt_data_extract.get_plain_text(channel_metadata['title']) + ' - About', - channel_title = yt_data_extract.get_plain_text(channel_metadata['title']), - avatar = html.escape(avatar), - description = description, - links = channel_links, - stats = stats, - channel_tabs = channel_tabs_html(channel_metadata['channelId'], 'About'), - ) + text = yt_data_extract.get_plain_text(link_json['title']) -def channel_search_page(polymer_json, query, current_page=1, number_of_videos = 1000, current_query_string=''): - response = polymer_json[1]['response'] - microformat = get_microformat(response) - channel_url = microformat['urlCanonical'].rstrip('/') - channel_id = channel_url[channel_url.rfind('/')+1:] + info['links'].append( (text, url) ) - - try: - items = tab_with_content(response['contents']['twoColumnBrowseResultsRenderer']['tabs'])['sectionListRenderer']['contents'] - except KeyError: - items = response['continuationContents']['sectionListContinuation']['contents'] - - items_html = list_items_html(items) - - return yt_channel_items_template.substitute( - header = html_common.get_header(), - channel_title = html.escape(microformat['title']), - channel_tabs = channel_tabs_html(channel_id, '', query), - avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'], - page_title = html.escape(query + ' - Channel search'), - items = items_html, - page_buttons = html_common.page_buttons_html(current_page, math.ceil(number_of_videos/29), util.URL_ORIGIN + "/channel/" + channel_id + "/search", current_query_string), - number_of_results = '', - sort_buttons = '', - ) -def get_channel_search_json(channel_id, query, page): - params = proto.string(2, 'search') + proto.string(15, str(page)) - params = proto.percent_b64encode(params) - ctoken = proto.string(2, channel_id) + proto.string(3, params) + proto.string(11, query) - ctoken = base64.urlsafe_b64encode(proto.nested(80226972, ctoken)).decode('ascii') - polymer_json = util.fetch_url("https://www.youtube.com/browse_ajax?ctoken=" + ctoken, util.desktop_ua + headers_1) - '''with open('debug/channel_search_debug', 'wb') as f: - f.write(polymer_json)''' - polymer_json = json.loads(polymer_json) + info['stats'] = [] + for stat_name in ('subscriberCountText', 'joinedDateText', 'viewCountText', 'country'): + try: + stat = channel_metadata[stat_name] + except KeyError: + continue + info['stats'].append(yt_data_extract.get_plain_text(stat)) + + + info['description'] = yt_data_extract.get_text(channel_metadata['description']) + + else: + raise NotImplementedError('Unknown or unsupported channel tab: ' + tab) + + + if html_prepare: + info['avatar'] = '/' + info['avatar'] + info['channel_url'] = '/' + info['channel_url'] + + + return info + - return polymer_json - playlist_sort_codes = {'2': "da", '3': "dd", '4': "lad"} -def get_channel_page(env, start_response): - path_parts = env['path_parts'] - channel_id = path_parts[1] - try: - tab = path_parts[2] - except IndexError: - tab = 'videos' - - parameters = env['parameters'] - page_number = int(util.default_multi_get(parameters, 'page', 0, default='1')) - sort = util.default_multi_get(parameters, 'sort', 0, default='3') - view = util.default_multi_get(parameters, 'view', 0, default='1') - query = util.default_multi_get(parameters, 'query', 0, default='') + +@yt_app.route('/channel//') +@yt_app.route('/channel//') +def get_channel_page(channel_id, tab='videos'): + + page_number = int(request.args.get('page', 1)) + sort = request.args.get('sort', '3') + view = request.args.get('view', '1') + query = request.args.get('query', '') + if tab == 'videos': tasks = ( @@ -397,17 +290,10 @@ def get_channel_page(env, start_response): gevent.joinall(tasks) number_of_videos, polymer_json = tasks[0].value, tasks[1].value - result = channel_videos_html(polymer_json, page_number, sort, number_of_videos, env['QUERY_STRING']) elif tab == 'about': polymer_json = util.fetch_url('https://www.youtube.com/channel/' + channel_id + '/about?pbj=1', util.desktop_ua + headers_1) - polymer_json = json.loads(polymer_json) - result = channel_about_page(polymer_json) elif tab == 'playlists': polymer_json = util.fetch_url('https://www.youtube.com/channel/' + channel_id + '/playlists?pbj=1&view=1&sort=' + playlist_sort_codes[sort], util.desktop_ua + headers_1) - '''with open('debug/channel_playlists_debug', 'wb') as f: - f.write(polymer_json)''' - polymer_json = json.loads(polymer_json) - result = channel_playlists_html(polymer_json, sort) elif tab == 'search': tasks = ( gevent.spawn(get_number_of_videos, channel_id ), @@ -416,54 +302,78 @@ def get_channel_page(env, start_response): gevent.joinall(tasks) number_of_videos, polymer_json = tasks[0].value, tasks[1].value - result = channel_search_page(polymer_json, query, page_number, number_of_videos, env['QUERY_STRING']) - else: - start_response('404 Not Found', [('Content-type', 'text/plain'),]) - return b'Unknown channel tab: ' + tab.encode('utf-8') - - start_response('200 OK', [('Content-type','text/html'),]) - return result.encode('utf-8') - -# youtube.com/user/[username]/[page] -# youtube.com/c/[custom]/[page] -# youtube.com/[custom]/[page] -def get_channel_page_general_url(env, start_response): - path_parts = env['path_parts'] - - is_toplevel = not path_parts[0] in ('user', 'c') - - if len(path_parts) + int(is_toplevel) == 3: # has /[page] after it - page = path_parts[2] - base_url = 'https://www.youtube.com/' + '/'.join(path_parts[0:-1]) - elif len(path_parts) + int(is_toplevel) == 2: # does not have /[page] after it, use /videos by default - page = 'videos' - base_url = 'https://www.youtube.com/' + '/'.join(path_parts) else: - start_response('404 Not Found', [('Content-type', 'text/plain'),]) - return b'Invalid channel url' + flask.abort(404, 'Unknown channel tab: ' + tab) - if page == 'videos': + + info = extract_info(json.loads(polymer_json), tab) + if tab in ('videos', 'search'): + info['number_of_videos'] = number_of_videos + info['number_of_pages'] = math.ceil(number_of_videos/30) + if tab in ('videos', 'playlists'): + info['current_sort'] = sort + elif tab == 'search': + info['search_box_value'] = query + + + return flask.render_template('channel.html', + parameters_dictionary = request.args, + **info + ) + + +# youtube.com/user/[username]/[tab] +# youtube.com/c/[custom]/[tab] +# youtube.com/[custom]/[tab] +def get_channel_page_general_url(base_url, tab, request): + + page_number = int(request.args.get('page', 1)) + sort = request.args.get('sort', '3') + view = request.args.get('view', '1') + query = request.args.get('query', '') + + if tab == 'videos': polymer_json = util.fetch_url(base_url + '/videos?pbj=1&view=0', util.desktop_ua + headers_1) - '''with open('debug/user_page_videos', 'wb') as f: - f.write(polymer_json)''' - polymer_json = json.loads(polymer_json) - result = channel_videos_html(polymer_json) - elif page == 'about': + with open('debug/channel_debug', 'wb') as f: + f.write(polymer_json) + elif tab == 'about': polymer_json = util.fetch_url(base_url + '/about?pbj=1', util.desktop_ua + headers_1) - polymer_json = json.loads(polymer_json) - result = channel_about_page(polymer_json) - elif page == 'playlists': + elif tab == 'playlists': polymer_json = util.fetch_url(base_url+ '/playlists?pbj=1&view=1', util.desktop_ua + headers_1) - polymer_json = json.loads(polymer_json) - result = channel_playlists_html(polymer_json) - elif page == 'search': + elif tab == 'search': raise NotImplementedError() - '''polymer_json = util.fetch_url('https://www.youtube.com/user' + username + '/search?pbj=1&' + query_string, util.desktop_ua + headers_1) - polymer_json = json.loads(polymer_json) - return channel_search_page(''' else: - start_response('404 Not Found', [('Content-type', 'text/plain'),]) - return b'Unknown channel page: ' + page.encode('utf-8') + flask.abort(404, 'Unknown channel tab: ' + tab) + + + info = extract_info(json.loads(polymer_json), tab) + if tab in ('videos', 'search'): + info['number_of_videos'] = 1000 + info['number_of_pages'] = math.ceil(1000/30) + if tab in ('videos', 'playlists'): + info['current_sort'] = sort + elif tab == 'search': + info['search_box_value'] = query + + + return flask.render_template('channel.html', + parameters_dictionary = request.args, + **info + ) + + +@yt_app.route('/user//') +@yt_app.route('/user//') +def get_user_page(username, tab='videos'): + return get_channel_page_general_url('https://www.youtube.com/user/' + username, tab, request) + +@yt_app.route('/c//') +@yt_app.route('/c//') +def get_custom_c_page(custom, tab='videos'): + return get_channel_page_general_url('https://www.youtube.com/c/' + custom, tab, request) + +@yt_app.route('/') +@yt_app.route('//') +def get_toplevel_custom_page(custom, tab='videos'): + return get_channel_page_general_url('https://www.youtube.com/' + custom, tab, request) - start_response('200 OK', [('Content-type','text/html'),]) - return result.encode('utf-8') diff --git a/youtube/templates/channel.html b/youtube/templates/channel.html new file mode 100644 index 0000000..8a3f279 --- /dev/null +++ b/youtube/templates/channel.html @@ -0,0 +1,144 @@ +{% extends "base.html" %} +{% block page_title %}{{ channel_name + ' - Channel' }}{% endblock %} +{% import "common_elements.html" as common_elements %} +{% block style %} + main{ + display:grid; +{% if current_tab == 'about' %} + grid-template-rows: 0fr 0fr 1fr; + grid-template-columns: 0fr 1fr; +{% else %} + grid-template-rows: repeat(5, 0fr); + grid-template-columns: auto 1fr; +{% endif %} + } + main .avatar{ + grid-row:1; + grid-column:1; + height:200px; + width:200px; + } + main .title{ + grid-row:1; + grid-column:2; + } + main .channel-tabs{ + grid-row:2; + grid-column: 1 / span 2; + + display:grid; + grid-auto-flow: column; + justify-content:start; + + background-color: #aaaaaa; + padding: 3px; + } + #links-metadata{ + display: grid; + grid-auto-flow: column; + grid-column-gap: 10px; + grid-column: 1/span 2; + justify-content: start; + padding-top: 8px; + padding-bottom: 8px; + background-color: #bababa; + margin-bottom: 10px; + } + #number-of-results{ + font-weight:bold; + } + .item-grid{ + grid-row:4; + grid-column: 1 / span 2; + } + .item-list{ + width:1000px; + grid-column: 1 / span 2; + } + .page-button-row{ + grid-column: 1 / span 2; + } + .tab{ + padding: 5px 75px; + } + main .channel-info{ + grid-row: 3; + grid-column: 1 / span 3; + } + .description{ + white-space: pre-wrap; + min-width: 0; + + } +{% endblock style %} + +{% block main %} + +

    {{ channel_name }}

    + + {% if current_tab == 'about' %} +
    +
      + {% for stat in stats %} +
    • {{ stat }}
      • + {% endfor %} +
    +
    +

    Description

    + {{ common_elements.text_runs(description) }} +
    +
      + {% for text, url in links %} +
    • {{ text }}
      • + {% endfor %} +
    +
    + {% else %} + + + {% if current_tab != 'about' %} + + + {% if current_tab != 'playlists' %} + + {% endif %} + {% endif %} + + {% endif %} +{% endblock main %} diff --git a/youtube/templates/common_elements.html b/youtube/templates/common_elements.html index b140332..0843c4b 100644 --- a/youtube/templates/common_elements.html +++ b/youtube/templates/common_elements.html @@ -14,7 +14,7 @@ {%- endif -%} {% endmacro %} -{% macro small_item(info) %} +{% macro small_item(info, include_author=true) %}
    {% if info['type'] == 'video' %} @@ -47,11 +47,13 @@
    {% endmacro %} -{% macro get_stats(info) %} - {% if 'author_url' is in(info) %} -
    By {{ info['author'] }}
    - {% else %} -
    {{ info['author'] }}
    +{% macro get_stats(info, include_author=true) %} + {% if include_author %} + {% if 'author_url' is in(info) %} +
    By {{ info['author'] }}
    + {% else %} +
    {{ info['author'] }}
    + {% endif %} {% endif %} {% if 'views' is in(info) %} {{ info['views'] }} @@ -63,7 +65,7 @@ -{% macro medium_item(info) %} +{% macro medium_item(info, include_author=true) %}
    {% if info['type'] == 'video' %} @@ -75,7 +77,7 @@ {{ info['title'] }}
    - {{ get_stats(info) }} + {{ get_stats(info, include_author) }}
    {{ text_runs(info.get('description', '')) }} @@ -91,7 +93,7 @@ {{ info['title'] }}
    - {{ get_stats(info) }} + {{ get_stats(info, include_author) }}
    {% elif info['type'] == 'channel' %} @@ -115,11 +117,11 @@ {% endmacro %} -{% macro item(info) %} +{% macro item(info, include_author=true) %} {% if info['item_size'] == 'small' %} - {{ small_item(info) }} + {{ small_item(info, include_author) }} {% elif info['item_size'] == 'medium' %} - {{ medium_item(info) }} + {{ medium_item(info, include_author) }} {% else %} Error: Unknown item size {% endif %} diff --git a/youtube/yt_data_extract.py b/youtube/yt_data_extract.py index a42b6a2..dca5964 100644 --- a/youtube/yt_data_extract.py +++ b/youtube/yt_data_extract.py @@ -36,19 +36,11 @@ import json - - def get_plain_text(node): try: - return html.escape(node['simpleText']) + return node['simpleText'] except KeyError: - return unformmated_text_runs(node['runs']) - -def unformmated_text_runs(runs): - result = '' - for text_run in runs: - result += html.escape(text_run["text"]) - return result + return ''.join(text_run['text'] for text_run in node['runs']) def format_text_runs(runs): if isinstance(runs, str): @@ -78,14 +70,19 @@ def get_url(node): def get_text(node): + if node == {}: + return '' try: return node['simpleText'] except KeyError: - pass + pass try: return node['runs'][0]['text'] except IndexError: # empty text runs return '' + except KeyError: + print(node) + raise def get_formatted_text(node): try: @@ -200,7 +197,7 @@ def renderer_info(renderer, additional_info={}): info.update(additional_info) - if type.startswith('compact') or type.startswith('playlist') or type.startswith('grid'): + if type.startswith('compact') or type.startswith('playlist'): info['item_size'] = 'small' else: info['item_size'] = 'medium' @@ -271,13 +268,8 @@ def renderer_info(renderer, additional_info={}): raise - - #print(renderer) - #raise NotImplementedError('Unknown renderer type: ' + type) - return '' - -def parse_info_prepare_for_html(renderer): - item = renderer_info(renderer) +def parse_info_prepare_for_html(renderer, additional_info={}): + item = renderer_info(renderer, additional_info) prefix_urls(item) add_extra_html_info(item) -- cgit v1.2.3 From c0617670f78bf61ccf3aa0c5904091146b630104 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 6 Jul 2019 19:26:07 -0700 Subject: Fix medium playlist items displaying incorrectly --- youtube/yt_data_extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/yt_data_extract.py b/youtube/yt_data_extract.py index dca5964..c236c2f 100644 --- a/youtube/yt_data_extract.py +++ b/youtube/yt_data_extract.py @@ -197,7 +197,7 @@ def renderer_info(renderer, additional_info={}): info.update(additional_info) - if type.startswith('compact') or type.startswith('playlist'): + if type.startswith('compact') or (type.startswith('playlist') and type != 'playlistRenderer'): info['item_size'] = 'small' else: info['item_size'] = 'medium' -- cgit v1.2.3 From 6261add37de1800c90b0942a77cca99baf0b1973 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 6 Jul 2019 23:08:04 -0700 Subject: Convert local_playlist to flask framework --- youtube/local_playlist.py | 106 +++++++++++++--------------- youtube/templates/local_playlist.html | 57 +++++++++++++++ youtube/templates/local_playlists_list.html | 16 +++++ 3 files changed, 121 insertions(+), 58 deletions(-) create mode 100644 youtube/templates/local_playlist.html create mode 100644 youtube/templates/local_playlists_list.html (limited to 'youtube') diff --git a/youtube/local_playlist.py b/youtube/local_playlist.py index e354013..8c673e9 100644 --- a/youtube/local_playlist.py +++ b/youtube/local_playlist.py @@ -1,5 +1,5 @@ -from youtube.template import Template -from youtube import util, html_common +from youtube import util, yt_data_extract +from youtube import yt_app import settings import os @@ -8,12 +8,12 @@ import html import gevent import urllib +import flask +from flask import request + playlists_directory = os.path.join(settings.data_dir, "playlists") thumbnails_directory = os.path.join(settings.data_dir, "playlist_thumbnails") -with open('yt_local_playlist_template.html', 'r', encoding='utf-8') as file: - local_playlist_template = Template(file.read()) - def video_ids_in_playlist(name): try: with open(os.path.join(playlists_directory, name + ".txt"), 'r', encoding='utf-8') as file: @@ -62,36 +62,34 @@ def download_thumbnails(playlist_name, ids): -def get_local_playlist_page(name): +def get_local_playlist_videos(name): try: thumbnails = set(os.listdir(os.path.join(thumbnails_directory, name))) except FileNotFoundError: thumbnails = set() missing_thumbnails = [] - videos_html = '' + videos = [] with open(os.path.join(playlists_directory, name + ".txt"), 'r', encoding='utf-8') as file: - videos = file.read() - videos = videos.splitlines() - for video in videos: + data = file.read() + videos_json = data.splitlines() + for video_json in videos_json: try: - info = json.loads(video) + info = json.loads(video_json) if info['id'] + ".jpg" in thumbnails: info['thumbnail'] = "/youtube.com/data/playlist_thumbnails/" + name + "/" + info['id'] + ".jpg" else: info['thumbnail'] = util.get_thumbnail_url(info['id']) missing_thumbnails.append(info['id']) - videos_html += html_common.video_item_html(info, html_common.small_video_item_template) + info['item_size'] = 'small' + info['type'] = 'video' + yt_data_extract.add_extra_html_info(info) + videos.append(info) except json.decoder.JSONDecodeError: - pass + if not video_json.strip() == '': + print('Corrupt playlist video entry: ' + video_json) gevent.spawn(download_thumbnails, name, missing_thumbnails) - return local_playlist_template.substitute( - page_title = name + ' - Local playlist', - header = html_common.get_header(), - videos = videos_html, - title = name, - page_buttons = '' - ) + return videos def get_playlist_names(): try: @@ -124,47 +122,39 @@ def remove_from_playlist(name, video_info_list): for file in to_delete: os.remove(os.path.join(thumbnails_directory, name, file)) -def get_playlists_list_page(): - page = '''\n''' - return html_common.yt_basic_template.substitute( - page_title = "Local playlists", - header = html_common.get_header(), - style = '', - page = page, - ) - - -def get_playlist_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - path_parts = env['path_parts'] - if len(path_parts) == 1: - return get_playlists_list_page().encode('utf-8') - else: - return get_local_playlist_page(path_parts[1]).encode('utf-8') -def path_edit_playlist(env, start_response): +@yt_app.route('/playlists', methods=['GET']) +@yt_app.route('/playlists/', methods=['GET']) +def get_local_playlist_page(playlist_name=None): + if playlist_name is None: + playlists = [(name, util.URL_ORIGIN + '/playlists/' + name) for name in get_playlist_names()] + return flask.render_template('local_playlists_list.html', playlists=playlists) + else: + videos = get_local_playlist_videos(playlist_name) + return flask.render_template('local_playlist.html', + playlist_name = playlist_name, + videos = videos, + ) + +@yt_app.route('/playlists/', methods=['POST']) +def path_edit_playlist(playlist_name): '''Called when making changes to the playlist from that playlist's page''' - parameters = env['parameters'] - if parameters['action'][0] == 'remove': - playlist_name = env['path_parts'][1] - remove_from_playlist(playlist_name, parameters['video_info_list']) - start_response('303 See Other', [('Location', util.URL_ORIGIN + env['PATH_INFO']),] ) - return b'' - + if request.values['action'] == 'remove': + remove_from_playlist(playlist_name, request.values.getlist('video_info_list')) + return flask.redirect(util.URL_ORIGIN + request.path) else: - start_response('400 Bad Request', [('Content-type', 'text/plain'),]) - return b'400 Bad Request' + flask.abort(400) -def edit_playlist(env, start_response): +@yt_app.route('/edit_playlist', methods=['POST']) +def edit_playlist(): '''Called when adding videos to a playlist from elsewhere''' - parameters = env['parameters'] - if parameters['action'][0] == 'add': - add_to_playlist(parameters['playlist_name'][0], parameters['video_info_list']) - start_response('204 No Content', ()) + if request.values['action'] == 'add': + add_to_playlist(request.values['playlist_name'], request.values.getlist('video_info_list')) + return '', 204 else: - start_response('400 Bad Request', [('Content-type', 'text/plain'),]) - return b'400 Bad Request' + flask.abort(400) + +@yt_app.route('/data/playlist_thumbnails//') +def serve_thumbnail(playlist_name, thumbnail): + # .. is necessary because flask always uses the application directory at ./youtube, not the working directory + return flask.send_from_directory(os.path.join('..', thumbnails_directory, playlist_name), thumbnail) diff --git a/youtube/templates/local_playlist.html b/youtube/templates/local_playlist.html new file mode 100644 index 0000000..27928df --- /dev/null +++ b/youtube/templates/local_playlist.html @@ -0,0 +1,57 @@ +{% extends "base.html" %} +{% block page_title %}{{ playlist_name + ' - Local playlist' }}{% endblock %} +{% import "common_elements.html" as common_elements %} +{% block style %} + main{ + display:grid; + grid-template-columns: 3fr 1fr; + } + + + + #left{ + grid-column: 1; + grid-row: 1; + + display: grid; + grid-template-columns: 1fr 800px auto; + grid-template-rows: 0fr 1fr 0fr; + } + .playlist-title{ + grid-column:2; + } + #playlist-remove-button{ + grid-column:3; + align-self: center; + white-space: nowrap; + } + #results{ + + grid-row: 2; + grid-column: 2 / span 2; + + + display: grid; + grid-auto-rows: 0fr; + grid-row-gap: 10px; + + } + .page-button-row{ + grid-row: 3; + grid-column: 2; + justify-self: center; + } +{% endblock style %} + +{% block main %} +
    +

    {{ playlist_name }}

    + + +
    + {% for video_info in videos %} + {{ common_elements.item(video_info) }} + {% endfor %} +
    +
    +{% endblock main %} diff --git a/youtube/templates/local_playlists_list.html b/youtube/templates/local_playlists_list.html new file mode 100644 index 0000000..269c151 --- /dev/null +++ b/youtube/templates/local_playlists_list.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{% block page_title %}Local playlists{% endblock %} + +{% block main %} + +{% endblock main %} + + + + + + -- cgit v1.2.3 From b89d90a0d35a239ee4eb476eaf5e2d3404fe65ea Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 7 Jul 2019 17:29:25 -0700 Subject: watch_page: refactor music list into flask template --- youtube/templates/watch.html | 21 ++++++++++++++++-- youtube/watch.py | 53 ++++++++++++-------------------------------- 2 files changed, 33 insertions(+), 41 deletions(-) (limited to 'youtube') diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html index 122958c..85c87ae 100644 --- a/youtube/templates/watch.html +++ b/youtube/templates/watch.html @@ -185,14 +185,31 @@ {{ format['resolution'] }} {{ format['note'] }} -{% endfor %} +{% endfor %}
    {{ description }}
    -{{ music_list|safe }} + {% if music_list.__len__() != 0 %} +
    + + + + {% for attribute in music_attributes %} + + {% endfor %} + + {% for track in music_list %} + + {% for attribute in music_attributes %} + + {% endfor %} + + {% endfor %} +
    Music
    {{ attribute }}
    {{ track.get(attribute.lower(), '') }}
    + {% endif %}
    {{ comments|safe }} diff --git a/youtube/watch.py b/youtube/watch.py index 1f7a352..6a1a199 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -110,46 +110,20 @@ def get_subtitle_sources(info): return sources -def get_music_list_html(music_list): - if len(music_list) == 0: - music_list_html = '' - else: - # get the set of attributes which are used by atleast 1 track - # so there isn't an empty, extraneous album column which no tracks use, for example - used_attributes = set() - for track in music_list: - used_attributes = used_attributes | track.keys() - - # now put them in the right order - ordered_attributes = [] - for attribute in ('Artist', 'Title', 'Album'): - if attribute.lower() in used_attributes: - ordered_attributes.append(attribute) - - music_list_html = '''
    - - - -''' - # table headings - for attribute in ordered_attributes: - music_list_html += "\n" - music_list_html += '''\n''' - - for track in music_list: - music_list_html += '''\n''' - for attribute in ordered_attributes: - try: - value = track[attribute.lower()] - except KeyError: - music_list_html += '''''' - else: - music_list_html += '''''' - music_list_html += '''\n''' - music_list_html += '''
    Music
    " + attribute + "
    ''' + html.escape(value) + '''
    \n''' - return music_list_html +def get_ordered_music_list_attributes(music_list): + # get the set of attributes which are used by atleast 1 track + # so there isn't an empty, extraneous album column which no tracks use, for example + used_attributes = set() + for track in music_list: + used_attributes = used_attributes | track.keys() + # now put them in the right order + ordered_attributes = [] + for attribute in ('Artist', 'Title', 'Album'): + if attribute.lower() in used_attributes: + ordered_attributes.append(attribute) + return ordered_attributes def extract_info(downloader, *args, **kwargs): @@ -231,10 +205,11 @@ def get_watch_page(): video_sources = get_video_sources(info), subtitle_sources = get_subtitle_sources(info), related = related_videos, + music_list = info['music_list'], + music_attributes = get_ordered_music_list_attributes(info['music_list']), # TODO: refactor these comments = comments_html, - music_list = get_music_list_html(info['music_list']), title = info['title'], uploader = info['uploader'], -- cgit v1.2.3 From 728b2b73d49d9112b8874783e038d901244dd7d8 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 7 Jul 2019 17:32:11 -0700 Subject: common_elements.html: Accept missing channel description as blank --- youtube/templates/common_elements.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/templates/common_elements.html b/youtube/templates/common_elements.html index 0843c4b..a6921e1 100644 --- a/youtube/templates/common_elements.html +++ b/youtube/templates/common_elements.html @@ -105,7 +105,7 @@ {{ info['subscriber_count'] }} {{ info['size'] }} - {{ text_runs(info['description']) }} + {{ text_runs(info.get('description', '')) }} {% else %} Error: unsupported item type {% endif %} -- cgit v1.2.3 From b854dab314267bd7f4677520b83a4e8abd0a1a6a Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 7 Jul 2019 17:46:07 -0700 Subject: watch.py regression: fix error page --- youtube/watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/watch.py b/youtube/watch.py index 6a1a199..1972cb4 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -156,7 +156,7 @@ def get_watch_page(): comments_html, info = tasks[0].value, tasks[1].value if isinstance(info, str): # youtube error - return flask.render_template('error.html', header = html_common.get_header, error_mesage = info) + return flask.render_template('error.html', error_message = info) video_info = { "duration": util.seconds_to_timestamp(info["duration"]), -- cgit v1.2.3 From 8cad77ad0d7e0a0d07629087e2ee1709688cb58d Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 19 Jul 2019 22:27:10 -0700 Subject: Convert comments to flask framework --- youtube/comments.py | 397 +++++++++++------------------------ youtube/templates/comments.html | 47 +++++ youtube/templates/comments_page.html | 83 ++++++++ youtube/templates/watch.html | 6 +- youtube/watch.py | 6 +- 5 files changed, 254 insertions(+), 285 deletions(-) create mode 100644 youtube/templates/comments.html create mode 100644 youtube/templates/comments_page.html (limited to 'youtube') diff --git a/youtube/comments.py b/youtube/comments.py index 94b086e..768bc13 100644 --- a/youtube/comments.py +++ b/youtube/comments.py @@ -1,57 +1,14 @@ -from youtube import proto, util, html_common, yt_data_extract, accounts +from youtube import proto, util, yt_data_extract, accounts +from youtube import yt_app import settings import json import base64 -from string import Template -import urllib.request import urllib -import html import re -comment_area_template = Template(''' -
    -$video-metadata -$comment-links -$comment-box -$comments -$more-comments-button -
    -''') -comment_template = Template(''' -
    -
    - -$avatar - -
    - $author -
    - - $text - - -
    -$replies -$action_buttons -
    -
    - -
    -''') -comment_avatar_template = Template(''' ''') - -reply_link_template = Template(''' - $view_replies_text -''') -with open("yt_comments_template.html", "r") as file: - yt_comments_template = Template(file.read()) - - -# $replies_link_text - +import flask +from flask import request # Here's what I know about the secret key (starting with ASJN_i) # *The secret key definitely contains the following information (or perhaps the information is stored at youtube's servers): @@ -102,6 +59,7 @@ def ctoken_metadata(ctoken): result['is_replies'] = False if (3 in offset_information) and (2 in proto.parse(offset_information[3])): result['is_replies'] = True + result['sort'] = None else: try: result['sort'] = proto.parse(offset_information[4])[6] @@ -109,12 +67,6 @@ def ctoken_metadata(ctoken): result['sort'] = 0 return result -def get_ids(ctoken): - params = proto.parse(proto.b64_to_bytes(ctoken)) - video_id = proto.parse(params[2])[2] - params = proto.parse(params[6]) - params = proto.parse(params[3]) - return params[2].decode('ascii'), video_id.decode('ascii') mobile_headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1', @@ -143,112 +95,65 @@ def request_comments(ctoken, replies=False): f.write(content)''' return content + def single_comment_ctoken(video_id, comment_id): page_params = proto.string(2, video_id) + proto.string(6, proto.percent_b64encode(proto.string(15, comment_id))) result = proto.nested(2, page_params) + proto.uint(3,6) return base64.urlsafe_b64encode(result).decode('ascii') - -def parse_comments_ajax(content, replies=False): - try: - content = json.loads(util.uppercase_escape(content.decode('utf-8'))) - #print(content) - comments_raw = content['content']['continuation_contents']['contents'] - ctoken = util.default_multi_get(content, 'content', 'continuation_contents', 'continuations', 0, 'continuation', default='') - - comments = [] - for comment_raw in comments_raw: - replies_url = '' - if not replies: - if comment_raw['replies'] is not None: - reply_ctoken = comment_raw['replies']['continuations'][0]['continuation'] - comment_id, video_id = get_ids(reply_ctoken) - replies_url = util.URL_ORIGIN + '/comments?parent_id=' + comment_id + "&video_id=" + video_id - comment_raw = comment_raw['comment'] - comment = { - 'author': comment_raw['author']['runs'][0]['text'], - 'author_url': comment_raw['author_endpoint']['url'], - 'author_channel_id': '', - 'author_id': '', - 'author_avatar': comment_raw['author_thumbnail']['url'], - 'likes': comment_raw['like_count'], - 'published': comment_raw['published_time']['runs'][0]['text'], - 'text': comment_raw['content']['runs'], - 'reply_count': '', - 'replies_url': replies_url, - } - comments.append(comment) - except Exception as e: - print('Error parsing comments: ' + str(e)) - comments = () - ctoken = '' - return {'ctoken': ctoken, 'comments': comments} -reply_count_regex = re.compile(r'(\d+)') -def parse_comments_polymer(content, replies=False): +def parse_comments_polymer(content): try: video_title = '' content = json.loads(util.uppercase_escape(content.decode('utf-8'))) url = content[1]['url'] ctoken = urllib.parse.parse_qs(url[url.find('?')+1:])['ctoken'][0] - video_id = ctoken_metadata(ctoken)['video_id'] - #print(content) + metadata = ctoken_metadata(ctoken) + try: comments_raw = content[1]['response']['continuationContents']['commentSectionContinuation']['items'] except KeyError: comments_raw = content[1]['response']['continuationContents']['commentRepliesContinuation']['contents'] - replies = True ctoken = util.default_multi_get(content, 1, 'response', 'continuationContents', 'commentSectionContinuation', 'continuations', 0, 'nextContinuationData', 'continuation', default='') - + comments = [] - for comment_raw in comments_raw: - replies_url = '' - view_replies_text = '' + for comment_json in comments_raw: + number_of_replies = 0 try: - comment_raw = comment_raw['commentThreadRenderer'] + comment_thread = comment_json['commentThreadRenderer'] except KeyError: - pass + comment_renderer = comment_json['commentRenderer'] else: - if 'commentTargetTitle' in comment_raw: - video_title = comment_raw['commentTargetTitle']['runs'][0]['text'] - - parent_id = comment_raw['comment']['commentRenderer']['commentId'] - # TODO: move this stuff into the comments_html function - if 'replies' in comment_raw: - #reply_ctoken = comment_raw['replies']['commentRepliesRenderer']['continuations'][0]['nextContinuationData']['continuation'] - #comment_id, video_id = get_ids(reply_ctoken) - replies_url = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id - view_replies_text = yt_data_extract.get_plain_text(comment_raw['replies']['commentRepliesRenderer']['moreText']) - match = reply_count_regex.search(view_replies_text) + if 'commentTargetTitle' in comment_thread: + video_title = comment_thread['commentTargetTitle']['runs'][0]['text'] + + if 'replies' in comment_thread: + view_replies_text = yt_data_extract.get_plain_text(comment_thread['replies']['commentRepliesRenderer']['moreText']) + view_replies_text = view_replies_text.replace(',', '') + match = re.search(r'(\d+)', view_replies_text) if match is None: - view_replies_text = '1 reply' + number_of_replies = 1 else: - view_replies_text = match.group(1) + " replies" - elif not replies: - view_replies_text = "Reply" - replies_url = util.URL_ORIGIN + '/post_comment?parent_id=' + parent_id + "&video_id=" + video_id - comment_raw = comment_raw['comment'] - - comment_raw = comment_raw['commentRenderer'] + number_of_replies = int(match.group(1)) + comment_renderer = comment_thread['comment']['commentRenderer'] + comment = { - 'author_id': comment_raw.get('authorId', ''), - 'author_avatar': comment_raw['authorThumbnail']['thumbnails'][0]['url'], - 'likes': comment_raw['likeCount'], - 'published': yt_data_extract.get_plain_text(comment_raw['publishedTimeText']), - 'text': comment_raw['contentText'].get('runs', ''), - 'view_replies_text': view_replies_text, - 'replies_url': replies_url, - 'video_id': video_id, - 'comment_id': comment_raw['commentId'], + 'author_id': comment_renderer.get('authorId', ''), + 'author_avatar': comment_renderer['authorThumbnail']['thumbnails'][0]['url'], + 'likes': comment_renderer['likeCount'], + 'published': yt_data_extract.get_plain_text(comment_renderer['publishedTimeText']), + 'text': comment_renderer['contentText'].get('runs', ''), + 'number_of_replies': number_of_replies, + 'comment_id': comment_renderer['commentId'], } - if 'authorText' in comment_raw: # deleted channels have no name or channel link - comment['author'] = yt_data_extract.get_plain_text(comment_raw['authorText']) - comment['author_url'] = comment_raw['authorEndpoint']['commandMetadata']['webCommandMetadata']['url'] - comment['author_channel_id'] = comment_raw['authorEndpoint']['browseEndpoint']['browseId'] + if 'authorText' in comment_renderer: # deleted channels have no name or channel link + comment['author'] = yt_data_extract.get_plain_text(comment_renderer['authorText']) + comment['author_url'] = comment_renderer['authorEndpoint']['commandMetadata']['webCommandMetadata']['url'] + comment['author_channel_id'] = comment_renderer['authorEndpoint']['browseEndpoint']['browseId'] else: comment['author'] = '' comment['author_url'] = '' @@ -260,172 +165,104 @@ def parse_comments_polymer(content, replies=False): comments = () ctoken = '' - return {'ctoken': ctoken, 'comments': comments, 'video_title': video_title} + return { + 'ctoken': ctoken, + 'comments': comments, + 'video_title': video_title, + 'video_id': metadata['video_id'], + 'offset': metadata['offset'], + 'is_replies': metadata['is_replies'], + 'sort': metadata['sort'], + } +def post_process_comments_info(comments_info): + for comment in comments_info['comments']: + comment['author_url'] = util.URL_ORIGIN + comment['author_url'] + comment['author_avatar'] = '/' + comment['author_avatar'] + comment['permalink'] = util.URL_ORIGIN + '/watch?v=' + comments_info['video_id'] + '&lc=' + comment['comment_id'] -def get_comments_html(comments): - html_result = '' - for comment in comments: - replies = '' - if comment['replies_url']: - replies = reply_link_template.substitute(url=comment['replies_url'], view_replies_text=html.escape(comment['view_replies_text'])) - if settings.enable_comment_avatars: - avatar = comment_avatar_template.substitute( - author_url = util.URL_ORIGIN + comment['author_url'], - author_avatar = '/' + comment['author_avatar'], - ) - else: - avatar = '' if comment['author_channel_id'] in accounts.accounts: - delete_url = (util.URL_ORIGIN + '/delete_comment?video_id=' - + comment['video_id'] + comment['delete_url'] = (util.URL_ORIGIN + '/delete_comment?video_id=' + + comments_info['video_id'] + '&channel_id='+ comment['author_channel_id'] + '&author_id=' + comment['author_id'] + '&comment_id=' + comment['comment_id']) - action_buttons = '''Delete''' + num_replies = comment['number_of_replies'] + if num_replies == 0: + comment['replies_url'] = util.URL_ORIGIN + '/post_comment?parent_id=' + comment['comment_id'] + "&video_id=" + comments_info['video_id'] else: - action_buttons = '' - - permalink = util.URL_ORIGIN + '/watch?v=' + comment['video_id'] + '&lc=' + comment['comment_id'] - html_result += comment_template.substitute( - author=comment['author'], - author_url = util.URL_ORIGIN + comment['author_url'], - avatar = avatar, - likes = str(comment['likes']) + ' likes' if str(comment['likes']) != '0' else '', - published = comment['published'], - text = yt_data_extract.format_text_runs(comment['text']), - datetime = '', #TODO - replies = replies, - action_buttons = action_buttons, - permalink = permalink, - ) - return html_result - + comment['replies_url'] = util.URL_ORIGIN + '/comments?parent_id=' + comment['comment_id'] + "&video_id=" + comments_info['video_id'] + + if num_replies == 0: + comment['view_replies_text'] = 'Reply' + elif num_replies == 1: + comment['view_replies_text'] = '1 reply' + else: + comment['view_replies_text'] = str(num_replies) + ' replies' + + + if comment['likes'] == 1: + comment['likes_text'] = '1 like' + else: + comment['likes_text'] = str(comment['likes']) + ' likes' + + comments_info['include_avatars'] = settings.enable_comment_avatars + if comments_info['ctoken'] != '': + comments_info['more_comments_url'] = util.URL_ORIGIN + '/comments?ctoken=' + comments_info['ctoken'] + + comments_info['page_number'] = page_number = str(int(comments_info['offset']/20) + 1) + + if not comments_info['is_replies']: + comments_info['sort_text'] = 'top' if comments_info['sort'] == 0 else 'newest' + + + comments_info['video_url'] = util.URL_ORIGIN + '/watch?v=' + comments_info['video_id'] + comments_info['video_thumbnail'] = '/i.ytimg.com/vi/'+ comments_info['video_id'] + '/mqdefault.jpg' + + def video_comments(video_id, sort=0, offset=0, lc='', secret_key=''): if settings.enable_comments: - post_comment_url = util.URL_ORIGIN + "/post_comment?video_id=" + video_id - post_comment_link = '''Post comment''' + comments_info = parse_comments_polymer(request_comments(make_comment_ctoken(video_id, sort, offset, lc, secret_key))) + post_process_comments_info(comments_info) + post_comment_url = util.URL_ORIGIN + "/post_comment?video_id=" + video_id other_sort_url = util.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(video_id, sort=1 - sort, lc=lc) - other_sort_name = 'newest' if sort == 0 else 'top' - other_sort_link = '''Sort by ''' + other_sort_name + '''''' - - comment_links = '''
    \n''' - comment_links += other_sort_link + '\n' + post_comment_link + '\n' - comment_links += '''
    ''' - - comment_info = parse_comments_polymer(request_comments(make_comment_ctoken(video_id, sort, offset, lc, secret_key))) - ctoken = comment_info['ctoken'] - - if ctoken == '': - more_comments_button = '' - else: - more_comments_button = more_comments_template.substitute(url = util.URL_ORIGIN + '/comments?ctoken=' + ctoken) - - result = '''
    \n''' - result += comment_links + '\n' - result += '
    \n' - result += get_comments_html(comment_info['comments']) + '\n' - result += '
    \n' - result += more_comments_button + '\n' - result += '''
    ''' - return result - return '' - -more_comments_template = Template('''More comments''') -video_metadata_template = Template('''
    - - - - $title - -

    Comments page $page_number

    - Sorted by $sort -
    -''') -account_option_template = Template(''' - ''') - -def comment_box_account_options(): - return ''.join(account_option_template.substitute(channel_id=channel_id, display_name=display_name) for channel_id, display_name in accounts.account_list_data()) - -comment_box_template = Template(''' -
    -
    - - - Add account -
    - - $video_id_input - -
    ''') -def get_comments_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),] ) - parameters = env['parameters'] - ctoken = util.default_multi_get(parameters, 'ctoken', 0, default='') + other_sort_text = 'Sort by ' + ('newest' if sort == 0 else 'top') + comments_info['comment_links'] = [('Post comment', post_comment_url), (other_sort_text, other_sort_url)] + + return comments_info + + return {} + + + +@yt_app.route('/comments') +def get_comments_page(): + ctoken = request.args.get('ctoken', '') replies = False if not ctoken: - video_id = parameters['video_id'][0] - parent_id = parameters['parent_id'][0] + video_id = request.args['video_id'] + parent_id = request.args['parent_id'] ctoken = comment_replies_ctoken(video_id, parent_id) replies = True - comment_info = parse_comments_polymer(request_comments(ctoken, replies), replies) + comments_info = parse_comments_polymer(request_comments(ctoken, replies)) + post_process_comments_info(comments_info) + + if not replies: + other_sort_url = util.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(comments_info['video_id'], sort=1 - comments_info['sort']) + other_sort_text = 'Sort by ' + ('newest' if comments_info['sort'] == 0 else 'top') + comments_info['comment_links'] = [(other_sort_text, other_sort_url)] + + + return flask.render_template('comments_page.html', + comments_info = comments_info, + + form_action = '' if replies else util.URL_ORIGIN + '/post_comment', + include_video_id_input = not replies, + accounts = accounts.account_list_data(), + ) - metadata = ctoken_metadata(ctoken) - if replies: - page_title = 'Replies' - video_metadata = '' - comment_box = comment_box_template.substitute(form_action='', video_id_input='', post_text='Post reply', options=comment_box_account_options()) - comment_links = '' - else: - page_number = str(int(metadata['offset']/20) + 1) - page_title = 'Comments page ' + page_number - - video_metadata = video_metadata_template.substitute( - page_number = page_number, - sort = 'top' if metadata['sort'] == 0 else 'newest', - title = html.escape(comment_info['video_title']), - url = util.URL_ORIGIN + '/watch?v=' + metadata['video_id'], - thumbnail = '/i.ytimg.com/vi/'+ metadata['video_id'] + '/mqdefault.jpg', - ) - comment_box = comment_box_template.substitute( - form_action= util.URL_ORIGIN + '/post_comment', - video_id_input='''''', - post_text='Post comment', - options=comment_box_account_options(), - ) - - other_sort_url = util.URL_ORIGIN + '/comments?ctoken=' + make_comment_ctoken(metadata['video_id'], sort=1 - metadata['sort']) - other_sort_name = 'newest' if metadata['sort'] == 0 else 'top' - other_sort_link = '''Sort by ''' + other_sort_name + '''''' - - - comment_links = '''
    \n''' - comment_links += other_sort_link + '\n' - comment_links += '''
    ''' - - comments_html = get_comments_html(comment_info['comments']) - ctoken = comment_info['ctoken'] - if ctoken == '': - more_comments_button = '' - else: - more_comments_button = more_comments_template.substitute(url = util.URL_ORIGIN + '/comments?ctoken=' + ctoken) - comments_area = '
    \n' - comments_area += video_metadata + comment_box + comment_links + '\n' - comments_area += '
    \n' - comments_area += comments_html + '\n' - comments_area += '
    \n' - comments_area += more_comments_button + '\n' - comments_area += '
    \n' - return yt_comments_template.substitute( - header = html_common.get_header(), - comments_area = comments_area, - page_title = page_title, - ).encode('utf-8') diff --git a/youtube/templates/comments.html b/youtube/templates/comments.html new file mode 100644 index 0000000..901190f --- /dev/null +++ b/youtube/templates/comments.html @@ -0,0 +1,47 @@ +{% import "common_elements.html" as common_elements %} + +{% macro render_comment(comment, include_avatar) %} +
    +
    + + {% if include_avatar %} + + {% endif %} + +
    + {{ comment['author'] }} +
    + + {{ common_elements.text_runs(comment['text']) }} + + +
    + {{ comment['view_replies_text'] }} + {% if 'delete_url' is in comment %} + Delete + {% endif %} +
    +
    + +
    +{% endmacro %} + +{% macro video_comments(comments_info) %} +
    +
    + {% for link_text, link_url in comments_info['comment_links'] %} + {{ link_text }} + {% endfor %} +
    +
    + {% for comment in comments_info['comments'] %} + {{ render_comment(comment, comments_info['include_avatars']) }} + {% endfor %} +
    + {% if 'more_comments_url' is in comments_info %} + More comments + {% endif %} +
    +{% endmacro %} diff --git a/youtube/templates/comments_page.html b/youtube/templates/comments_page.html new file mode 100644 index 0000000..a77d2b4 --- /dev/null +++ b/youtube/templates/comments_page.html @@ -0,0 +1,83 @@ +{% extends "base.html" %} +{% import "comments.html" as comments %} + +{% block page_title %}{{ 'Replies' if comments_info['is_replies'] else 'Comments page ' + comments_info['page_number'] }}{% endblock %} + + +{% block style %} + main{ + display:grid; + grid-template-columns: 3fr 2fr; + } + #left{ + background-color:#bcbcbc; + + display: grid; + grid-column: 1; + grid-row: 1; + grid-template-columns: 1fr 640px; + grid-template-rows: 0fr 0fr 0fr; + } + .comments-area{ + grid-column:2; + } + .comment{ + width:640px; + } +{% endblock style %} + + +{% block main %} +
    +
    + {% if not comments_info['is_replies'] %} +
    + + + + {{ comments_info['video_title'] }} + +

    Comments page {{ comments_info['page_number'] }}

    + Sorted by {{ comments_info['sort_text'] }} +
    + {% endif %} + + +
    +
    + + + Add account +
    + + {% if include_video_id_input %} + + {% endif %} + +
    + + {% if not comments_info['is_replies'] %} +
    + {% for link_text, link_url in comments_info['comment_links'] %} + {{ link_text }} + {% endfor %} +
    + {% endif %} + +
    + {% for comment in comments_info['comments'] %} + {{ comments.render_comment(comment, comments_info['include_avatars']) }} + {% endfor %} +
    + {% if 'more_comments_url' is in comments_info %} + More comments + {% endif %} +
    +
    +{% endblock main %} + + diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html index 85c87ae..d61997f 100644 --- a/youtube/templates/watch.html +++ b/youtube/templates/watch.html @@ -1,5 +1,6 @@ {% extends "base.html" %} {% import "common_elements.html" as common_elements %} +{% import "comments.html" as comments %} {% block page_title %}{{ title }}{% endblock %} {% block style %} main{ @@ -211,7 +212,10 @@ {% endif %}
    -{{ comments|safe }} + + {% if comments_info %} + {{ comments.video_comments(comments_info) }} + {% endif %} diff --git a/youtube/watch.py b/youtube/watch.py index 1972cb4..dc50b7b 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -153,7 +153,7 @@ def get_watch_page(): gevent.spawn(extract_info, yt_dl_downloader, "https://www.youtube.com/watch?v=" + video_id, download=False) ) gevent.joinall(tasks) - comments_html, info = tasks[0].value, tasks[1].value + comments_info, info = tasks[0].value, tasks[1].value if isinstance(info, str): # youtube error return flask.render_template('error.html', error_message = info) @@ -207,9 +207,7 @@ def get_watch_page(): related = related_videos, music_list = info['music_list'], music_attributes = get_ordered_music_list_attributes(info['music_list']), - - # TODO: refactor these - comments = comments_html, + comments_info = comments_info, title = info['title'], uploader = info['uploader'], -- cgit v1.2.3 From 167483af21fb252622170c4b5e3ae3ce81f58733 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 00:03:34 -0700 Subject: Convert login to flask framework and fix the login, and fix account display --- youtube/accounts.py | 102 +++++++---------------------------- youtube/templates/comments_page.html | 2 +- youtube/templates/login.html | 60 +++++++++++++++++++++ 3 files changed, 79 insertions(+), 85 deletions(-) create mode 100644 youtube/templates/login.html (limited to 'youtube') diff --git a/youtube/accounts.py b/youtube/accounts.py index 375bf2a..c35b6cc 100644 --- a/youtube/accounts.py +++ b/youtube/accounts.py @@ -1,5 +1,6 @@ # Contains functions having to do with logging in -from youtube import util, html_common +from youtube import util +from youtube import yt_app import settings import urllib @@ -9,6 +10,9 @@ import http.cookiejar import io import os +import flask +from flask import request + try: with open(os.path.join(settings.data_dir, 'accounts.txt'), 'r', encoding='utf-8') as f: accounts = json.loads(f.read()) @@ -18,7 +22,7 @@ except FileNotFoundError: def account_list_data(): '''Returns iterable of (channel_id, account_display_name)''' - return ( (channel_id, account['display_name']) for channel_id, account in accounts.items() ) + return [ (channel_id, account['display_name']) for channel_id, account in accounts.items() ] def save_accounts(): to_save = {channel_id: account for channel_id, account in accounts.items() if account['save']} @@ -51,91 +55,20 @@ def _add_account(username, password, save, use_tor): return True return False -def add_account(env, start_response): - parameters = env['parameters'] - if 'save' in parameters and parameters['save'][0] == "on": - save_account = True - else: - save_account = False +@yt_app.route('/login', methods=['POST']) +def add_account(): + save_account = request.values.get('save', 'off') == 'on' + use_tor = request.values.get('use_tor', 'off') == 'on' - if 'use_tor' in parameters and parameters['use_tor'][0] == "on": - use_tor = True + if _add_account(request.values['username'], request.values['password'], save_account, use_tor ): + return 'Account successfully added' else: - use_tor = False + return 'Failed to add account' - if _add_account(parameters['username'][0], parameters['password'][0], save_account, use_tor ): - start_response('200 OK', [('Content-type', 'text/plain'),] ) - return b'Account successfully added' - else: - start_response('200 OK', [('Content-type', 'text/plain'),] ) - return b'Failed to add account' - -def get_account_login_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),] ) - - style = ''' - main{ - display: grid; - grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); - align-content: start; - grid-row-gap: 40px; - } - - main form{ - margin-top:20px; - grid-column:2; - display:grid; - justify-items: start; - align-content: start; - grid-row-gap: 10px; - } - - #username, #password{ - grid-column:2; - width: 250px; - } - #add-account-button{ - margin-top:20px; - } - #tor-note{ - grid-row:2; - grid-column:2; - background-color: #dddddd; - padding: 10px; - } - ''' - - page = ''' -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    - -
    -
    Note on using Tor to log in
    -Using Tor to log in should only be done if the account was created using a proxy/VPN/Tor to begin with and hasn't been logged in using your IP. Otherwise, it's pointless since Google already knows who the account belongs to. When logging into a google account, it must be logged in using an IP address geographically close to the area where the account was created or where it is logged into regularly. If the account was created using an IP address in America and is logged into from an IP in Russia, Google will block the Russian IP from logging in, assume someone knows your password, lock the account, and make you change your password. If creating an account using Tor, you must remember the IP (or geographic region) it was created in, and only log in using that geographic region for the exit node. This can be accomplished by putting the desired IP in the torrc file to force Tor to use that exit node. Using the login cookie to post comments through Tor is perfectly safe, however. -
    - ''' - - return html_common.yt_basic_template.substitute( - page_title = "Login", - style = style, - header = html_common.get_header(), - page = page, - ).encode('utf-8') + +@yt_app.route('/login', methods=['GET']) +def get_account_login_page(): + return flask.render_template('login.html') @@ -249,6 +182,7 @@ def _login(username, password, cookiejar, use_tor): 'f.req': json.dumps(f_req), 'flowName': 'GlifWebSignIn', 'flowEntry': 'ServiceLogin', + 'bgRequest': '["identifier",""]', }) headers={ 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', diff --git a/youtube/templates/comments_page.html b/youtube/templates/comments_page.html index a77d2b4..302fcac 100644 --- a/youtube/templates/comments_page.html +++ b/youtube/templates/comments_page.html @@ -48,7 +48,7 @@ Add account diff --git a/youtube/templates/login.html b/youtube/templates/login.html new file mode 100644 index 0000000..49a5365 --- /dev/null +++ b/youtube/templates/login.html @@ -0,0 +1,60 @@ +{% extends "base.html" %} +{% block page_title %}Login{% endblock %} + +{% block style %} + main{ + display: grid; + grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); + align-content: start; + grid-row-gap: 40px; + } + + main form{ + margin-top:20px; + grid-column:2; + display:grid; + justify-items: start; + align-content: start; + grid-row-gap: 10px; + } + + #username, #password{ + grid-column:2; + width: 250px; + } + #add-account-button{ + margin-top:20px; + } + #tor-note{ + grid-row:2; + grid-column:2; + background-color: #dddddd; + padding: 10px; + } +{% endblock style %} + +{% block main %} +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    Note on using Tor to log in
    +Using Tor to log in should only be done if the account was created using a proxy/VPN/Tor to begin with and hasn't been logged in using your IP. Otherwise, it's pointless since Google already knows who the account belongs to. When logging into a google account, it must be logged in using an IP address geographically close to the area where the account was created or where it is logged into regularly. If the account was created using an IP address in America and is logged into from an IP in Russia, Google will block the Russian IP from logging in, assume someone knows your password, lock the account, and make you change your password. If creating an account using Tor, you must remember the IP (or geographic region) it was created in, and only log in using that geographic region for the exit node. This can be accomplished by putting the desired IP in the torrc file to force Tor to use that exit node. Using the login cookie to post comments through Tor is perfectly safe, however. +
    +{% endblock main %} + -- cgit v1.2.3 From fc295ac93d0ad6b0272aa94f2d2ea44002ecbc48 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 21:48:54 -0700 Subject: Convert comment posting system to flask framework --- youtube/comments.py | 13 ++- youtube/post_comment.py | 149 ++++++++++++++-------------------- youtube/templates/comments.html | 23 ++++++ youtube/templates/comments_page.html | 18 +--- youtube/templates/delete_comment.html | 26 ++++++ youtube/templates/post_comment.html | 30 +++++++ 6 files changed, 148 insertions(+), 111 deletions(-) create mode 100644 youtube/templates/delete_comment.html create mode 100644 youtube/templates/post_comment.html (limited to 'youtube') diff --git a/youtube/comments.py b/youtube/comments.py index 768bc13..ba82154 100644 --- a/youtube/comments.py +++ b/youtube/comments.py @@ -258,11 +258,16 @@ def get_comments_page(): comments_info['comment_links'] = [(other_sort_text, other_sort_url)] + comment_posting_box_info = { + 'form_action': '' if replies else util.URL_ORIGIN + '/post_comment', + 'video_id': comments_info['video_id'], + 'accounts': accounts.account_list_data(), + 'include_video_id_input': not replies, + 'replying': replies, + } + return flask.render_template('comments_page.html', comments_info = comments_info, - - form_action = '' if replies else util.URL_ORIGIN + '/post_comment', - include_video_id_input = not replies, - accounts = accounts.account_list_data(), + comment_posting_box_info = comment_posting_box_info, ) diff --git a/youtube/post_comment.py b/youtube/post_comment.py index 876a1c0..26899c2 100644 --- a/youtube/post_comment.py +++ b/youtube/post_comment.py @@ -1,5 +1,6 @@ # Contains functions having to do with posting/editing/deleting comments -from youtube import util, html_common, proto, comments, accounts +from youtube import util, proto, comments, accounts +from youtube import yt_app import settings import urllib @@ -8,6 +9,9 @@ import re import traceback import os +import flask +from flask import request + def _post_comment(text, video_id, session_token, cookiejar): headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1', @@ -109,108 +113,73 @@ def get_session_token(video_id, cookiejar): else: raise Exception("Couldn't find xsrf_token") -def delete_comment(env, start_response): - parameters = env['parameters'] - video_id = parameters['video_id'][0] - cookiejar = accounts.account_cookiejar(parameters['channel_id'][0]) +@yt_app.route('/delete_comment', methods=['POST']) +def delete_comment(): + video_id = request.values['video_id'] + cookiejar = accounts.account_cookiejar(request.values['channel_id']) token = get_session_token(video_id, cookiejar) - code = _delete_comment(video_id, parameters['comment_id'][0], parameters['author_id'][0], token, cookiejar) + code = _delete_comment(video_id, request.values['comment_id'], request.values['author_id'], token, cookiejar) if code == "SUCCESS": - start_response('303 See Other', [('Location', util.URL_ORIGIN + '/comment_delete_success'),] ) + return flask.redirect(util.URL_ORIGIN + '/comment_delete_success', 303) else: - start_response('303 See Other', [('Location', util.URL_ORIGIN + '/comment_delete_fail'),] ) + return flask.redirect(util.URL_ORIGIN + '/comment_delete_fail', 303) + +@yt_app.route('/comment_delete_success') +def comment_delete_success(): + return 'Successfully deleted comment' -def post_comment(env, start_response): - parameters = env['parameters'] - video_id = parameters['video_id'][0] - channel_id = parameters['channel_id'][0] +@yt_app.route('/comment_delete_fail') +def comment_delete_fail(): + return 'Failed to delete comment' + +@yt_app.route('/post_comment', methods=['POST']) +@yt_app.route('/comments', methods=['POST']) +def post_comment(): + video_id = request.values['video_id'] + channel_id = request.values['channel_id'] cookiejar = accounts.account_cookiejar(channel_id) token = get_session_token(video_id, cookiejar) - if 'parent_id' in parameters: - code = _post_comment_reply(parameters['comment_text'][0], parameters['video_id'][0], parameters['parent_id'][0], token, cookiejar) - start_response('303 See Other', (('Location', util.URL_ORIGIN + '/comments?' + env['QUERY_STRING']),) ) - + if 'parent_id' in request.values: + code = _post_comment_reply(request.values['comment_text'], request.values['video_id'], request.values['parent_id'], token, cookiejar) + return flask.redirect(util.URL_ORIGIN + '/comments?' + request.query_string.decode('utf-8'), 303) else: - code = _post_comment(parameters['comment_text'][0], parameters['video_id'][0], token, cookiejar) - start_response('303 See Other', (('Location', util.URL_ORIGIN + '/comments?ctoken=' + comments.make_comment_ctoken(video_id, sort=1)),) ) + code = _post_comment(request.values['comment_text'], request.values['video_id'], token, cookiejar) + return flask.redirect(util.URL_ORIGIN + '/comments?ctoken=' + comments.make_comment_ctoken(video_id, sort=1), 303) - return b'' +@yt_app.route('/delete_comment', methods=['GET']) +def get_delete_comment_page(): + parameters = [(parameter_name, request.args[parameter_name]) for parameter_name in ('video_id', 'channel_id', 'author_id', 'comment_id')] + return flask.render_template('delete_comment.html', parameters = parameters) -def get_delete_comment_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - parameters = env['parameters'] - style = ''' - main{ - display: grid; - grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); - align-content: start; - } - main > div, main > form{ - margin-top:20px; - grid-column:2; - } - ''' - - page = ''' -
    Are you sure you want to delete this comment?
    -
    ''' - for parameter in ('video_id', 'channel_id', 'author_id', 'comment_id'): - page += '''\n ''' - page += ''' - -
    ''' - return html_common.yt_basic_template.substitute( - page_title = "Delete comment?", - style = style, - header = html_common.get_header(), - page = page, - ).encode('utf-8') - -def get_post_comment_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - parameters = env['parameters'] - video_id = parameters['video_id'][0] - parent_id = util.default_multi_get(parameters, 'parent_id', 0, default='') +@yt_app.route('/post_comment', methods=['GET']) +def get_post_comment_page(): + video_id = request.args['video_id'] + parent_id = request.args.get('parent_id', '') - style = ''' main{ - display: grid; - grid-template-columns: 3fr 2fr; -} -.left{ - display:grid; - grid-template-columns: 1fr 640px; -} -textarea{ - width: 460px; - height: 85px; -} -.comment-form{ - grid-column:2; - justify-content:start; -}''' if parent_id: # comment reply - comment_box = comments.comment_box_template.substitute( - form_action = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id, - video_id_input = '', - post_text = "Post reply", - options=comments.comment_box_account_options(), - ) + form_action = util.URL_ORIGIN + '/comments?parent_id=' + parent_id + "&video_id=" + video_id + replying = True else: - comment_box = comments.comment_box_template.substitute( - form_action = util.URL_ORIGIN + '/post_comment', - video_id_input = '''''', - post_text = "Post comment", - options=comments.comment_box_account_options(), - ) - - page = '''
    \n''' + comment_box + '''
    \n''' - return html_common.yt_basic_template.substitute( - page_title = "Post comment reply" if parent_id else "Post a comment", - style = style, - header = html_common.get_header(), - page = page, - ).encode('utf-8') + form_action = '' + replying = False + + + comment_posting_box_info = { + 'form_action': form_action, + 'video_id': video_id, + 'accounts': accounts.account_list_data(), + 'include_video_id_input': not replying, + 'replying': replying, + } + return flask.render_template('post_comment.html', + comment_posting_box_info = comment_posting_box_info, + replying = replying, + ) + + + + diff --git a/youtube/templates/comments.html b/youtube/templates/comments.html index 901190f..82276b8 100644 --- a/youtube/templates/comments.html +++ b/youtube/templates/comments.html @@ -45,3 +45,26 @@ {% endif %} {% endmacro %} + +{% macro comment_posting_box(info) %} +
    +
    + + + Add account +
    + + {% if info['include_video_id_input'] %} + + {% endif %} + +
    +{% endmacro %} + + + + diff --git a/youtube/templates/comments_page.html b/youtube/templates/comments_page.html index 302fcac..c7947fa 100644 --- a/youtube/templates/comments_page.html +++ b/youtube/templates/comments_page.html @@ -42,23 +42,7 @@ {% endif %} - -
    -
    - - - Add account -
    - - {% if include_video_id_input %} - - {% endif %} - -
    + {{ comments.comment_posting_box(comment_posting_box_info) }} {% if not comments_info['is_replies'] %}
    diff --git a/youtube/templates/delete_comment.html b/youtube/templates/delete_comment.html new file mode 100644 index 0000000..4797240 --- /dev/null +++ b/youtube/templates/delete_comment.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block page_title %}Delete comment?{% endblock %} + +{% block style %} + main{ + display: grid; + grid-template-columns: minmax(0px, 3fr) 640px 40px 500px minmax(0px,2fr); + align-content: start; + } + main > div, main > form{ + margin-top:20px; + grid-column:2; + } +{% endblock style %} + +{% block main %} +
    Are you sure you want to delete this comment?
    +
    + {% for parameter_name, parameter_value in parameters %} + + {% endfor %} + +
    +{% endblock %} + diff --git a/youtube/templates/post_comment.html b/youtube/templates/post_comment.html new file mode 100644 index 0000000..98316ab --- /dev/null +++ b/youtube/templates/post_comment.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% import "comments.html" as comments %} + +{% block page_title %}{{ 'Post reply' if replying else 'Post comment' }}{% endblock %} + +{% block style %} + main{ + display: grid; + grid-template-columns: 3fr 2fr; + } + .left{ + display:grid; + grid-template-columns: 1fr 640px; + } + textarea{ + width: 460px; + height: 85px; + } + .comment-form{ + grid-column:2; + justify-content:start; + } +{% endblock style %} + +{% block main %} +
    + {{ comments.comment_posting_box(comment_posting_box_info) }} +
    +{% endblock %} + -- cgit v1.2.3 From 8eff0bb9e23690bb8a87453a4a6d4d415b9e1a95 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 21:59:52 -0700 Subject: Delete obsolete files --- youtube/channel.py | 3 +- youtube/html_common.py | 270 ------------------------------------------------- youtube/template.py | 132 ------------------------ youtube/watch.py | 2 +- 4 files changed, 2 insertions(+), 405 deletions(-) delete mode 100644 youtube/html_common.py delete mode 100644 youtube/template.py (limited to 'youtube') diff --git a/youtube/channel.py b/youtube/channel.py index 9cb1e78..17bc48d 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -1,8 +1,7 @@ import base64 -from youtube import util, yt_data_extract, html_common +from youtube import util, yt_data_extract from youtube import yt_app -import http_errors import urllib import json from string import Template diff --git a/youtube/html_common.py b/youtube/html_common.py deleted file mode 100644 index b8ea0d6..0000000 --- a/youtube/html_common.py +++ /dev/null @@ -1,270 +0,0 @@ -from youtube.template import Template -from youtube import local_playlist, yt_data_extract, util - -import json -import html - - -with open('yt_basic_template.html', 'r', encoding='utf-8') as file: - yt_basic_template = Template(file.read()) - - - - -page_button_template = Template('''$page''') -current_page_button_template = Template('''
    $page''') - -medium_playlist_item_template = Template(''' -
    -
    - - -
    - $size -
    -     - - $title - -
    $stats
    -
    -
    -''') -medium_video_item_template = Template(''' -
    -
    - - - $duration - - - $title - -
    $stats
    - - $description - $badges -
    - -
    -''') - -small_video_item_template = Template(''' -
    -
    - - - $duration - - $title - -
    $author
    - $views - -
    - -
    -''') - -small_playlist_item_template = Template(''' -
    -
    - - -
    - $size -
    -     - $title - -
    $author
    -
    -
    -''') - -medium_channel_item_template = Template(''' -
    -
    - - - $duration - - - $title - - $subscriber_count - $size - - $description -
    -
    -''') - - - - - -def badges_html(badges): - return ' | '.join(map(html.escape, badges)) - - -html_transform_dispatch = { - 'title': html.escape, - 'published': html.escape, - 'id': html.escape, - 'description': yt_data_extract.format_text_runs, - 'duration': html.escape, - 'thumbnail': lambda url: html.escape('/' + url.lstrip('/')), - 'size': html.escape, - 'author': html.escape, - 'author_url': lambda url: html.escape(util.URL_ORIGIN + url), - 'views': html.escape, - 'subscriber_count': html.escape, - 'badges': badges_html, - 'playlist_index': html.escape, -} - -def get_html_ready(item): - html_ready = {} - for key, value in item.items(): - try: - function = html_transform_dispatch[key] - except KeyError: - continue - html_ready[key] = function(value) - return html_ready - - -author_template_url = Template('''
    By $author
    ''') -author_template = Template('''
    $author
    ''') -stat_templates = ( - Template('''$views'''), - Template(''''''), -) -def get_stats(html_ready): - stats = [] - if 'author' in html_ready: - if 'author_url' in html_ready: - stats.append(author_template_url.substitute(html_ready)) - else: - stats.append(author_template.substitute(html_ready)) - for stat in stat_templates: - try: - stats.append(stat.strict_substitute(html_ready)) - except KeyError: - pass - return ' | '.join(stats) - -def video_item_html(item, template, html_exclude=set()): - - video_info = {} - for key in ('id', 'title', 'author'): - try: - video_info[key] = item[key] - except KeyError: - video_info[key] = '' - try: - video_info['duration'] = item['duration'] - except KeyError: - video_info['duration'] = 'Live' # livestreams don't have a duration - - html_ready = get_html_ready(item) - - html_ready['video_info'] = html.escape(json.dumps(video_info) ) - html_ready['url'] = util.URL_ORIGIN + "/watch?v=" + html_ready['id'] - html_ready['datetime'] = '' #TODO - - for key in html_exclude: - del html_ready[key] - html_ready['stats'] = get_stats(html_ready) - - return template.substitute(html_ready) - - -def playlist_item_html(item, template, html_exclude=set()): - html_ready = get_html_ready(item) - - html_ready['url'] = util.URL_ORIGIN + "/playlist?list=" + html_ready['id'] - html_ready['datetime'] = '' #TODO - - for key in html_exclude: - del html_ready[key] - html_ready['stats'] = get_stats(html_ready) - - return template.substitute(html_ready) - - - - - - - -page_button_template = Template('''$page''') -current_page_button_template = Template('''
    $page
    ''') - -def page_buttons_html(current_page, estimated_pages, url, current_query_string): - if current_page <= 5: - page_start = 1 - page_end = min(9, estimated_pages) - else: - page_start = current_page - 4 - page_end = min(current_page + 4, estimated_pages) - - result = "" - for page in range(page_start, page_end+1): - if page == current_page: - template = current_page_button_template - else: - template = page_button_template - result += template.substitute(page=page, href = url + "?" + util.update_query_string(current_query_string, {'page': [str(page)]}) ) - return result - - - - - - - -showing_results_for = Template(''' -
    -
    Showing results for $corrected_query
    -
    Search instead for $original_query
    -
    -''') - -did_you_mean = Template(''' -
    -
    Did you mean $corrected_query
    -
    -''') - -def renderer_html(renderer, additional_info={}, current_query_string=''): - type = list(renderer.keys())[0] - renderer = renderer[type] - if type == 'itemSectionRenderer': - return renderer_html(renderer['contents'][0], additional_info, current_query_string) - - if type == 'channelRenderer': - info = yt_data_extract.renderer_info(renderer) - html_ready = get_html_ready(info) - html_ready['url'] = util.URL_ORIGIN + "/channel/" + html_ready['id'] - return medium_channel_item_template.substitute(html_ready) - - if type in ('movieRenderer', 'clarificationRenderer'): - return '' - - info = yt_data_extract.renderer_info(renderer) - info.update(additional_info) - html_exclude = set(additional_info.keys()) - if type == 'compactVideoRenderer': - return video_item_html(info, small_video_item_template, html_exclude=html_exclude) - if type in ('compactPlaylistRenderer', 'compactRadioRenderer', 'compactShowRenderer'): - return playlist_item_html(info, small_playlist_item_template, html_exclude=html_exclude) - if type in ('videoRenderer', 'gridVideoRenderer'): - return video_item_html(info, medium_video_item_template, html_exclude=html_exclude) - if type in ('playlistRenderer', 'gridPlaylistRenderer', 'radioRenderer', 'gridRadioRenderer', 'gridShowRenderer', 'showRenderer'): - return playlist_item_html(info, medium_playlist_item_template, html_exclude=html_exclude) - - #print(renderer) - #raise NotImplementedError('Unknown renderer type: ' + type) - return '' \ No newline at end of file diff --git a/youtube/template.py b/youtube/template.py deleted file mode 100644 index d1a5e58..0000000 --- a/youtube/template.py +++ /dev/null @@ -1,132 +0,0 @@ - -import re as _re -from collections import ChainMap as _ChainMap - -class _TemplateMetaclass(type): - pattern = r""" - %(delim)s(?: - (?P%(delim)s) | # Escape sequence of two delimiters - (?P%(id)s) | # delimiter and a Python identifier - {(?P%(id)s)} | # delimiter and a braced identifier - (?P) # Other ill-formed delimiter exprs - ) - """ - - def __init__(cls, name, bases, dct): - super(_TemplateMetaclass, cls).__init__(name, bases, dct) - if 'pattern' in dct: - pattern = cls.pattern - else: - pattern = _TemplateMetaclass.pattern % { - 'delim' : _re.escape(cls.delimiter), - 'id' : cls.idpattern, - } - cls.pattern = _re.compile(pattern, cls.flags | _re.VERBOSE) - - -class Template(metaclass=_TemplateMetaclass): - """A string class for supporting $-substitutions.""" - - delimiter = '$' - idpattern = r'[_a-z][_a-z0-9]*' - flags = _re.IGNORECASE - - def __init__(self, template): - self.template = template - - # Search for $$, $identifier, ${identifier}, and any bare $'s - - def _invalid(self, mo): - i = mo.start('invalid') - lines = self.template[:i].splitlines(keepends=True) - if not lines: - colno = 1 - lineno = 1 - else: - colno = i - len(''.join(lines[:-1])) - lineno = len(lines) - raise ValueError('Invalid placeholder in string: line %d, col %d' % - (lineno, colno)) - - def substitute(*args, **kws): - if not args: - raise TypeError("descriptor 'substitute' of 'Template' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: - mapping = kws - elif kws: - mapping = _ChainMap(kws, args[0]) - else: - mapping = args[0] - # Helper function for .sub() - def convert(mo): - # Check the most common path first. - named = mo.group('named') or mo.group('braced') - if named is not None: - return str(mapping.get(named,'')) - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - self._invalid(mo) - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - - def strict_substitute(*args, **kws): - if not args: - raise TypeError("descriptor 'substitute' of 'Template' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: - mapping = kws - elif kws: - mapping = _ChainMap(kws, args[0]) - else: - mapping = args[0] - # Helper function for .sub() - def convert(mo): - # Check the most common path first. - named = mo.group('named') or mo.group('braced') - if named is not None: - return str(mapping[named]) - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - self._invalid(mo) - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - - def safe_substitute(*args, **kws): - if not args: - raise TypeError("descriptor 'safe_substitute' of 'Template' object " - "needs an argument") - self, *args = args # allow the "self" keyword be passed - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: - mapping = kws - elif kws: - mapping = _ChainMap(kws, args[0]) - else: - mapping = args[0] - # Helper function for .sub() - def convert(mo): - named = mo.group('named') or mo.group('braced') - if named is not None: - try: - return str(mapping[named]) - except KeyError: - return mo.group() - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - return mo.group() - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) diff --git a/youtube/watch.py b/youtube/watch.py index dc50b7b..3b71e75 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -1,5 +1,5 @@ from youtube import yt_app -from youtube import util, html_common, comments, local_playlist, yt_data_extract +from youtube import util, comments, local_playlist, yt_data_extract import settings from flask import request -- cgit v1.2.3 From 2837d9db2b757b1ca54c4523e2644cee6b3798f2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 22:16:38 -0700 Subject: Convert captioning to flask --- youtube/watch.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'youtube') diff --git a/youtube/watch.py b/youtube/watch.py index 3b71e75..e324115 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -214,3 +214,14 @@ def get_watch_page(): description = info['description'], unlisted = info['unlisted'], ) + + +@yt_app.route('/api/') +def get_captions(dummy): + result = util.fetch_url('https://www.youtube.com' + request.full_path) + result = result.replace(b"align:start position:0%", b"") + return result + + + + -- cgit v1.2.3 From 86382706fe87afc63b2757a1a133497c75ce3cd2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 22:28:57 -0700 Subject: Add flask Firefox search engine route --- youtube/search.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/search.py b/youtube/search.py index 76a814c..39a80bf 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -1,10 +1,12 @@ from youtube import util, yt_data_extract, proto, local_playlist from youtube import yt_app +import settings import json import urllib import base64 from math import ceil +import mimetypes from flask import request import flask @@ -124,4 +126,8 @@ def get_search_page(): parameters_dictionary = request.args, ) - +@yt_app.route('/opensearch.xml') +def get_search_engine_xml(): + with open("youtube/opensearch.xml", 'rb') as f: + content = f.read().replace(b'$port_number', str(settings.port_number).encode()) + return flask.Response(content, mimetype='application/xml') -- cgit v1.2.3 From c5827a3bb1a6b2eb6cfbc7d9104bd73514311246 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 22:43:29 -0700 Subject: Add status template, use for comment deletion status --- youtube/post_comment.py | 4 ++-- youtube/templates/status.html | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 youtube/templates/status.html (limited to 'youtube') diff --git a/youtube/post_comment.py b/youtube/post_comment.py index 26899c2..c4ffb9d 100644 --- a/youtube/post_comment.py +++ b/youtube/post_comment.py @@ -128,11 +128,11 @@ def delete_comment(): @yt_app.route('/comment_delete_success') def comment_delete_success(): - return 'Successfully deleted comment' + return flask.render_template('status.html', title='Success', message='Successfully deleted comment') @yt_app.route('/comment_delete_fail') def comment_delete_fail(): - return 'Failed to delete comment' + return flask.render_template('status.html', title='Error', message='Failed to delete comment') @yt_app.route('/post_comment', methods=['POST']) @yt_app.route('/comments', methods=['POST']) diff --git a/youtube/templates/status.html b/youtube/templates/status.html new file mode 100644 index 0000000..1e18b5c --- /dev/null +++ b/youtube/templates/status.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block page_title %}{{ title if (title is defined) else 'Status'}}{% endblock %} + +{% block main %} + {{ message }} +{% endblock %} + -- cgit v1.2.3 From 74cf3b2135661ae17b4e390c91182c673645cad7 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 22:50:57 -0700 Subject: Use variable to set title on base template instead of block --- youtube/templates/base.html | 2 +- youtube/templates/channel.html | 2 +- youtube/templates/comments_page.html | 4 +--- youtube/templates/delete_comment.html | 3 +-- youtube/templates/error.html | 3 +-- youtube/templates/local_playlist.html | 2 +- youtube/templates/local_playlists_list.html | 2 +- youtube/templates/login.html | 2 +- youtube/templates/playlist.html | 2 +- youtube/templates/post_comment.html | 3 +-- youtube/templates/search.html | 2 +- youtube/templates/status.html | 3 +-- youtube/templates/watch.html | 2 +- 13 files changed, 13 insertions(+), 19 deletions(-) (limited to 'youtube') diff --git a/youtube/templates/base.html b/youtube/templates/base.html index eafd369..3dda55d 100644 --- a/youtube/templates/base.html +++ b/youtube/templates/base.html @@ -2,7 +2,7 @@ - {% block page_title %}{{ title }}{% endblock %} + {{ page_title }} diff --git a/youtube/templates/channel.html b/youtube/templates/channel.html index 8a3f279..734769d 100644 --- a/youtube/templates/channel.html +++ b/youtube/templates/channel.html @@ -1,5 +1,5 @@ +{% set page_title = channel_name + ' - Channel' %} {% extends "base.html" %} -{% block page_title %}{{ channel_name + ' - Channel' }}{% endblock %} {% import "common_elements.html" as common_elements %} {% block style %} main{ diff --git a/youtube/templates/comments_page.html b/youtube/templates/comments_page.html index c7947fa..68c8537 100644 --- a/youtube/templates/comments_page.html +++ b/youtube/templates/comments_page.html @@ -1,9 +1,7 @@ +{% set page_title = ('Replies' if comments_info['is_replies'] else 'Comments page ' + comments_info['page_number']) %} {% extends "base.html" %} {% import "comments.html" as comments %} -{% block page_title %}{{ 'Replies' if comments_info['is_replies'] else 'Comments page ' + comments_info['page_number'] }}{% endblock %} - - {% block style %} main{ display:grid; diff --git a/youtube/templates/delete_comment.html b/youtube/templates/delete_comment.html index 4797240..71555ee 100644 --- a/youtube/templates/delete_comment.html +++ b/youtube/templates/delete_comment.html @@ -1,7 +1,6 @@ +{% set page_title = 'Delete comment?' %} {% extends "base.html" %} -{% block page_title %}Delete comment?{% endblock %} - {% block style %} main{ display: grid; diff --git a/youtube/templates/error.html b/youtube/templates/error.html index 1f33c44..e77c92c 100644 --- a/youtube/templates/error.html +++ b/youtube/templates/error.html @@ -1,7 +1,6 @@ +{% set page_title = 'Error' %} {% extends "base.html" %} -{% block page_title %}Error{% endblock %} - {% block main %} {{ error_message }} {% endblock %} diff --git a/youtube/templates/local_playlist.html b/youtube/templates/local_playlist.html index 27928df..a7564ca 100644 --- a/youtube/templates/local_playlist.html +++ b/youtube/templates/local_playlist.html @@ -1,5 +1,5 @@ +{% set page_title = playlist_name + ' - Local playlist' %} {% extends "base.html" %} -{% block page_title %}{{ playlist_name + ' - Local playlist' }}{% endblock %} {% import "common_elements.html" as common_elements %} {% block style %} main{ diff --git a/youtube/templates/local_playlists_list.html b/youtube/templates/local_playlists_list.html index 269c151..9b5f510 100644 --- a/youtube/templates/local_playlists_list.html +++ b/youtube/templates/local_playlists_list.html @@ -1,5 +1,5 @@ +{% set page_title = 'Local playlists' %} {% extends "base.html" %} -{% block page_title %}Local playlists{% endblock %} {% block main %}
      diff --git a/youtube/templates/login.html b/youtube/templates/login.html index 49a5365..0f09a62 100644 --- a/youtube/templates/login.html +++ b/youtube/templates/login.html @@ -1,5 +1,5 @@ +{% set page_title = 'Login' %} {% extends "base.html" %} -{% block page_title %}Login{% endblock %} {% block style %} main{ diff --git a/youtube/templates/playlist.html b/youtube/templates/playlist.html index 09e382b..074b813 100644 --- a/youtube/templates/playlist.html +++ b/youtube/templates/playlist.html @@ -1,5 +1,5 @@ +{% set page_title = title + ' - Page ' + parameters_dictionary.get('page', '1') %} {% extends "base.html" %} -{% block page_title %}{{ title + ' - Page ' + parameters_dictionary.get('page', '1') }}{% endblock %} {% import "common_elements.html" as common_elements %} {% block style %} main{ diff --git a/youtube/templates/post_comment.html b/youtube/templates/post_comment.html index 98316ab..67c54f1 100644 --- a/youtube/templates/post_comment.html +++ b/youtube/templates/post_comment.html @@ -1,8 +1,7 @@ +{% set page_title = 'Post reply' if replying else 'Post comment' %} {% extends "base.html" %} {% import "comments.html" as comments %} -{% block page_title %}{{ 'Post reply' if replying else 'Post comment' }}{% endblock %} - {% block style %} main{ display: grid; diff --git a/youtube/templates/search.html b/youtube/templates/search.html index 1086cfd..782a85e 100644 --- a/youtube/templates/search.html +++ b/youtube/templates/search.html @@ -1,6 +1,6 @@ {% set search_box_value = query %} +{% set page_title = query + ' - Search' %} {% extends "base.html" %} -{% block page_title %}{{ query + ' - Search' }}{% endblock %} {% import "common_elements.html" as common_elements %} {% block style %} main{ diff --git a/youtube/templates/status.html b/youtube/templates/status.html index 1e18b5c..901aa5b 100644 --- a/youtube/templates/status.html +++ b/youtube/templates/status.html @@ -1,7 +1,6 @@ +{% set page_title = (title if (title is defined) else 'Status') %} {% extends "base.html" %} -{% block page_title %}{{ title if (title is defined) else 'Status'}}{% endblock %} - {% block main %} {{ message }} {% endblock %} diff --git a/youtube/templates/watch.html b/youtube/templates/watch.html index d61997f..82c1a97 100644 --- a/youtube/templates/watch.html +++ b/youtube/templates/watch.html @@ -1,7 +1,7 @@ +{% set page_title = title %} {% extends "base.html" %} {% import "common_elements.html" as common_elements %} {% import "comments.html" as comments %} -{% block page_title %}{{ title }}{% endblock %} {% block style %} main{ display:grid; -- cgit v1.2.3 From f165eb5e29fee1fa4d3d9980d7402e2b6cc9825e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 22 Jul 2019 00:20:38 -0700 Subject: opensearch.xml: add port number in the other search URL just in case --- youtube/opensearch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/opensearch.xml b/youtube/opensearch.xml index fce9d71..9f035a6 100644 --- a/youtube/opensearch.xml +++ b/youtube/opensearch.xml @@ -4,7 +4,7 @@ UTF-8 data:image/x-icon;base64,AAABAAEAEBAAAAEACAAlAgAAFgAAAIlQTkcNChoKAAAADUlIRFIAAAAQAAAAEAgGAAAAH/P/YQAAAexJREFUOI2lkzFPmlEUhp/73fshtCUCRtvQkJoKMrDQJvoHnBzUhc3EH0DUQf+As6tujo4M6mTiIDp0kGiMTRojTRNSW6o12iD4YYXv3g7Qr4O0ScM7npz7vOe+J0fk83lDF7K6eQygwkdHhI+P0bYNxmBXq5RmZui5vGQgn0f7fKi7O4oLC1gPD48BP9JpnpRKJFZXcQMB3m1u4vr9NHp76d/bo39/n4/z84ROThBa4/r91OJxMKb9BSn5mskAIOt1eq6uEFpjVyrEcjk+T0+TXlzkbTZLuFDAur9/nIFRipuREQCe7+zgBgK8mZvj/fIylVTKa/6UzXKbSnnuHkA0GnwbH/cA0a0takND3IyOEiwWAXBiMYTWjzLwtvB9bAyAwMUF8ZUVPiwtYTWbHqA6PIxoNv8OMLbN3eBga9TZWYQxaKX+AJJJhOv+AyAlT0slAG6TSX5n8+zszJugkzxA4PzcK9YSCQCk42DXaq1aGwqgfT5ebG9jpMQyUjKwu8vrtbWWqxC83NjAd31NsO2uleJnX58HCJ6eEjk8BGNQAA+RCOXJScpTU2AMwnUxlkXk4ACA+2iUSKGArNeRjkMsl6M8MYHQGtHpmIxSvFpfRzoORinQGqvZBCEwQoAxfMlkaIRCnQH/o66v8Re19MavaDNLfgAAAABJRU5ErkJggg== - + http://localhost:$port_number/youtube.com/search -- cgit v1.2.3 From 80b325a22561758c4ef93e8d8a0c29db9de703bd Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 22 Jul 2019 00:53:22 -0700 Subject: Add homepage route --- youtube/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'youtube') diff --git a/youtube/__init__.py b/youtube/__init__.py index faab170..e620827 100644 --- a/youtube/__init__.py +++ b/youtube/__init__.py @@ -1,3 +1,7 @@ import flask yt_app = flask.Flask(__name__) yt_app.url_map.strict_slashes = False + +@yt_app.route('/') +def homepage(): + return flask.render_template('base.html', title="Youtube local") -- cgit v1.2.3 From dfbe29355ff4c2dd106efb5acb9240e5292b0f08 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 22 Jul 2019 01:01:27 -0700 Subject: Channel: fix oversized thumbnails in firefox quantum --- youtube/templates/channel.html | 3 +++ 1 file changed, 3 insertions(+) (limited to 'youtube') diff --git a/youtube/templates/channel.html b/youtube/templates/channel.html index 734769d..ffcf793 100644 --- a/youtube/templates/channel.html +++ b/youtube/templates/channel.html @@ -70,6 +70,9 @@ min-width: 0; } + .medium-item img{ + max-width: 168px; + } {% endblock style %} {% block main %} -- cgit v1.2.3 From 6a00cfab1b932b0c5570cf4c52ac2171d2ed3f4e Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 22 Jul 2019 01:04:43 -0700 Subject: playlist: fix incorrect display of description --- youtube/templates/playlist.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube') diff --git a/youtube/templates/playlist.html b/youtube/templates/playlist.html index 074b813..371b51b 100644 --- a/youtube/templates/playlist.html +++ b/youtube/templates/playlist.html @@ -85,7 +85,7 @@
      {{ views }}
      {{ size }}
    -
    {{ description }}
    +
    {{ common_elements.text_runs(description) }}
    -- cgit v1.2.3 From 2034b6bd251baf52998ce8a81c7016f91e711302 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 23 Jul 2019 22:34:13 -0700 Subject: Fix unrouted channel avatar image on empty channels --- youtube/channel.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'youtube') diff --git a/youtube/channel.py b/youtube/channel.py index 17bc48d..fe5af8c 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -141,7 +141,7 @@ def get_channel_search_json(channel_id, query, page): return polymer_json -def extract_info(polymer_json, tab, html_prepare=True): +def extract_info(polymer_json, tab): response = polymer_json[1]['response'] try: microformat = response['microformat']['microformatDataRenderer'] @@ -174,10 +174,10 @@ def extract_info(polymer_json, tab, html_prepare=True): info['channel_id'] = channel_id info['channel_url'] = 'https://www.youtube.com/channel/' + channel_id + info['items'] = [] # empty channel if 'contents' not in response and 'continuationContents' not in response: - info['items'] = [] return info @@ -225,10 +225,7 @@ def extract_info(polymer_json, tab, html_prepare=True): # TODO: Fix this URL prefixing shit additional_info = {'author': info['channel_name'], 'author_url': '/channel/' + channel_id} - if html_prepare: - info['items'] = [yt_data_extract.parse_info_prepare_for_html(renderer, additional_info) for renderer in items] - elif items is not None: - info['items'] = [yt_data_extract.renderer_info(renderer, additional_info) for renderer in items] + info['items'] = [yt_data_extract.renderer_info(renderer, additional_info) for renderer in items] elif tab == 'about': channel_metadata = tab_content['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['channelAboutFullMetadataRenderer'] @@ -260,13 +257,17 @@ def extract_info(polymer_json, tab, html_prepare=True): else: raise NotImplementedError('Unknown or unsupported channel tab: ' + tab) + return info + +def post_process_channel_info(info): + info['avatar'] = '/' + info['avatar'] + info['channel_url'] = '/' + info['channel_url'] + for item in info['items']: + yt_data_extract.prefix_urls(item) + yt_data_extract.add_extra_html_info(item) - if html_prepare: - info['avatar'] = '/' + info['avatar'] - info['channel_url'] = '/' + info['channel_url'] - return info playlist_sort_codes = {'2': "da", '3': "dd", '4': "lad"} @@ -306,6 +307,7 @@ def get_channel_page(channel_id, tab='videos'): info = extract_info(json.loads(polymer_json), tab) + post_process_channel_info(info) if tab in ('videos', 'search'): info['number_of_videos'] = number_of_videos info['number_of_pages'] = math.ceil(number_of_videos/30) @@ -346,6 +348,7 @@ def get_channel_page_general_url(base_url, tab, request): info = extract_info(json.loads(polymer_json), tab) + post_process_channel_info(info) if tab in ('videos', 'search'): info['number_of_videos'] = 1000 info['number_of_pages'] = math.ceil(1000/30) -- cgit v1.2.3 From fe38aa214bc8876e80de08d3f3f793fa2f546ef4 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 23 Jul 2019 22:36:43 -0700 Subject: Fix error when channel has empty description --- youtube/channel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'youtube') diff --git a/youtube/channel.py b/youtube/channel.py index fe5af8c..4f719ae 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -251,8 +251,10 @@ def extract_info(polymer_json, tab): continue info['stats'].append(yt_data_extract.get_plain_text(stat)) - - info['description'] = yt_data_extract.get_text(channel_metadata['description']) + if 'description' in channel_metadata: + info['description'] = yt_data_extract.get_text(channel_metadata['description']) + else: + info['description'] = '' else: raise NotImplementedError('Unknown or unsupported channel tab: ' + tab) -- cgit v1.2.3 From cb1c899a4570e8644dad572c7f00a3d96c844a2f Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 23 Jul 2019 22:55:15 -0700 Subject: channel: Add short description under channel name --- youtube/channel.py | 2 +- youtube/templates/channel.html | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'youtube') diff --git a/youtube/channel.py b/youtube/channel.py index 4f719ae..04f698b 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -166,7 +166,7 @@ def extract_info(polymer_json, tab): # stuff from microformat (info given by youtube for every page on channel) - info['description'] = microformat['description'] + info['short_description'] = microformat['description'] info['channel_name'] = microformat['title'] info['avatar'] = microformat['thumbnail']['thumbnails'][0]['url'] channel_url = microformat['urlCanonical'].rstrip('/') diff --git a/youtube/templates/channel.html b/youtube/templates/channel.html index ffcf793..8293559 100644 --- a/youtube/templates/channel.html +++ b/youtube/templates/channel.html @@ -18,7 +18,7 @@ height:200px; width:200px; } - main .title{ + main .summary{ grid-row:1; grid-column:2; } @@ -77,7 +77,10 @@ {% block main %} -

    {{ channel_name }}

    +
    +

    {{ channel_name }}

    +

    {{ short_description }}

    +
    + {% endblock main %} -- cgit v1.2.3