aboutsummaryrefslogtreecommitdiffstats
path: root/youtube
diff options
context:
space:
mode:
authorJames Taylor <user234683@users.noreply.github.com>2019-02-16 23:41:52 -0800
committerJames Taylor <user234683@users.noreply.github.com>2019-02-16 23:41:52 -0800
commit3905e7e64059b45479894ba1fdfb0ef9cef64475 (patch)
tree4c5dbbfd204d0351cac8412cc87a65fea49c1a52 /youtube
parent24642455d0dc5841ddec99f456598c4f763c1e8a (diff)
downloadyt-local-3905e7e64059b45479894ba1fdfb0ef9cef64475.tar.lz
yt-local-3905e7e64059b45479894ba1fdfb0ef9cef64475.tar.xz
yt-local-3905e7e64059b45479894ba1fdfb0ef9cef64475.zip
basic subscriptions system
Diffstat (limited to 'youtube')
-rw-r--r--youtube/channel.py4
-rw-r--r--youtube/subscriptions.py71
-rw-r--r--youtube/youtube.py2
3 files changed, 64 insertions, 13 deletions
diff --git a/youtube/channel.py b/youtube/channel.py
index 9577525..c83d7d1 100644
--- a/youtube/channel.py
+++ b/youtube/channel.py
@@ -248,6 +248,7 @@ def channel_videos_html(polymer_json, current_page=1, current_sort=3, number_of_
return yt_channel_items_template.substitute(
header = common.get_header(),
channel_title = microformat['title'],
+ channel_id = channel_id,
channel_tabs = channel_tabs_html(channel_id, 'Videos'),
sort_buttons = channel_sort_buttons_html(channel_id, 'videos', current_sort),
avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'],
@@ -269,6 +270,7 @@ def channel_playlists_html(polymer_json, current_sort=3):
return yt_channel_items_template.substitute(
header = common.get_header(),
channel_title = microformat['title'],
+ channel_id = channel_id,
channel_tabs = channel_tabs_html(channel_id, 'Playlists'),
sort_buttons = channel_sort_buttons_html(channel_id, 'playlists', current_sort),
avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'],
@@ -333,6 +335,7 @@ def channel_about_page(polymer_json):
description = description,
links = channel_links,
stats = stats,
+ channel_id = channel_metadata['channelId'],
channel_tabs = channel_tabs_html(channel_metadata['channelId'], 'About'),
)
@@ -353,6 +356,7 @@ def channel_search_page(polymer_json, query, current_page=1, number_of_videos =
return yt_channel_items_template.substitute(
header = common.get_header(),
channel_title = html.escape(microformat['title']),
+ channel_id = channel_id,
channel_tabs = channel_tabs_html(channel_id, '', query),
avatar = '/' + microformat['thumbnail']['thumbnails'][0]['url'],
page_title = html.escape(query + ' - Channel search'),
diff --git a/youtube/subscriptions.py b/youtube/subscriptions.py
index 82916dd..ff7d0df 100644
--- a/youtube/subscriptions.py
+++ b/youtube/subscriptions.py
@@ -5,6 +5,10 @@ import sqlite3
import os
import secrets
import datetime
+import itertools
+import time
+import urllib
+import socks, sockshandler
# so as to not completely break on people who have updated but don't know of new dependency
try:
@@ -51,11 +55,16 @@ def open_database():
return connection
-def _subscribe(channel_id, channel_name):
+def _subscribe(channels):
+ ''' channels is a list of (channel_id, channel_name) '''
+
+ # set time_last_checked to 0 on all channels being subscribed to
+ channels = ( (channel_id, channel_name, 0) for channel_id, channel_name in channels)
+
connection = open_database()
try:
cursor = connection.cursor()
- cursor.execute("INSERT INTO subscribed_channels (channel_id, name) VALUES (?, ?)", (channel_id, channel_name))
+ cursor.executemany("INSERT INTO subscribed_channels (channel_id, channel_name, time_last_checked) VALUES (?, ?, ?)", channels)
connection.commit()
except:
connection.rollback()
@@ -63,11 +72,12 @@ def _subscribe(channel_id, channel_name):
finally:
connection.close()
-def _unsubscribe(channel_id):
+def _unsubscribe(channel_ids):
+ ''' channel_ids is a list of channel_ids '''
connection = open_database()
try:
cursor = connection.cursor()
- cursor.execute("DELETE FROM subscribed_channels WHERE channel_id=?", (channel_id, ))
+ cursor.executemany("DELETE FROM subscribed_channels WHERE channel_id=?", ((channel_id, ) for channel_id in channel_ids))
connection.commit()
except:
connection.rollback()
@@ -125,12 +135,14 @@ def youtube_timestamp_to_posix(dumb_timestamp):
weekdays = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
-def _get_upstream_videos(channel_id, channel_name, time_last_checked):
+def _get_upstream_videos(channel_id, time_last_checked):
feed_url = "https://www.youtube.com/feeds/videos.xml?channel_id=" + channel_id
headers = {}
# randomly change time_last_checked up to one day earlier to make tracking harder
time_last_checked = time_last_checked - secrets.randbelow(24*3600)
+ if time_last_checked < 0: # happens when time_last_checked is initialized to 0 when checking for first time
+ time_last_checked = 0
# If-Modified-Since header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since
struct_time = time.gmtime(time_last_checked)
@@ -142,7 +154,7 @@ def _get_upstream_videos(channel_id, channel_name, time_last_checked):
headers['User-Agent'] = 'Python-urllib' # Don't leak python version
headers['Accept-Encoding'] = 'gzip, br'
- req = urllib.request.Request(url, headers=headers)
+ req = urllib.request.Request(feed_url, headers=headers)
if settings.route_tor:
opener = urllib.request.build_opener(sockshandler.SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150))
else:
@@ -165,13 +177,10 @@ def _get_upstream_videos(channel_id, channel_name, time_last_checked):
for entry in feed.entries:
video_id = entry.id_[9:] # example of id_: yt:video:q6EoRBvdVPQ
- # standard names used in this program for purposes of html templating
atom_videos[video_id] = {
'title': entry.title.value,
- 'author': entry.authors[0].name,
#'description': '', # Not supported by atoma
#'duration': '', # Youtube's atom feeds don't provide it.. very frustrating
- 'published': entry.published.strftime('%m/%d/%Y'),
'time_published': int(entry.published.timestamp()),
}
@@ -182,12 +191,13 @@ def _get_upstream_videos(channel_id, channel_name, time_last_checked):
# Now check channel page to retrieve missing information for videos
json_channel_videos = channel.get_grid_items(channel.get_channel_tab(channel_id)[1]['response'])
for json_video in json_channel_videos:
- info = renderer_info(json_video)
+ info = common.renderer_info(json_video['gridVideoRenderer'])
+ if 'description' not in info:
+ info['description'] = ''
if info['id'] in atom_videos:
info.update(atom_videos[info['id']])
else:
- info['author'] = channel_name
- info['time published'] = youtube_timestamp_to_posix(info['published'])
+ info['time_published'] = youtube_timestamp_to_posix(info['published'])
videos.append(info)
return videos
@@ -195,7 +205,7 @@ def get_subscriptions_page(env, start_response):
items_html = '''<nav class="item-grid">\n'''
for item in _get_videos(30, 0):
- items_html += common.video_item_html(info, common.small_video_item_template)
+ items_html += common.video_item_html(item, common.small_video_item_template)
items_html += '''\n</nav>'''
start_response('200 OK', [('Content-type','text/html'),])
@@ -205,3 +215,38 @@ def get_subscriptions_page(env, start_response):
page_buttons = '',
).encode('utf-8')
+def post_subscriptions_page(env, start_response):
+ params = env['parameters']
+ action = params['action'][0]
+ if action == 'subscribe':
+ if len(params['channel_id']) != len(params['channel_name']):
+ start_response('400 Bad Request', ())
+ return b'400 Bad Request, length of channel_id != length of channel_name'
+ _subscribe(zip(params['channel_id'], params['channel_name']))
+
+ elif action == 'unsubscribe':
+ _unsubscribe(params['channel_id'])
+
+ elif action == 'refresh':
+ connection = open_database()
+ try:
+ cursor = connection.cursor()
+ for uploader_id, channel_id, time_last_checked in cursor.execute('''SELECT id, channel_id, time_last_checked FROM subscribed_channels'''):
+ db_videos = ( (uploader_id, info['id'], info['title'], info['duration'], info['time_published'], info['description']) for info in _get_upstream_videos(channel_id, time_last_checked) )
+ cursor.executemany('''INSERT INTO videos (uploader_id, video_id, title, duration, time_published, description) VALUES (?, ?, ?, ?, ?, ?)''', db_videos)
+
+ cursor.execute('''UPDATE subscribed_channels SET time_last_checked = ?''', ( int(time.time()), ) )
+ connection.commit()
+ except:
+ connection.rollback()
+ raise
+ finally:
+ connection.close()
+
+ start_response('303 See Other', [('Location', common.URL_ORIGIN + '/subscriptions'),] )
+ return b''
+ else:
+ start_response('400 Bad Request', ())
+ return b'400 Bad Request'
+ start_response('204 No Content', ())
+ return b''
diff --git a/youtube/youtube.py b/youtube/youtube.py
index ad73a6e..288f68b 100644
--- a/youtube/youtube.py
+++ b/youtube/youtube.py
@@ -35,6 +35,8 @@ post_handlers = {
'comments': post_comment.post_comment,
'post_comment': post_comment.post_comment,
'delete_comment': post_comment.delete_comment,
+
+ 'subscriptions': subscriptions.post_subscriptions_page,
}
def youtube(env, start_response):