aboutsummaryrefslogtreecommitdiffstats
path: root/youtube
diff options
context:
space:
mode:
authorJames Taylor <user234683@users.noreply.github.com>2019-08-11 17:09:38 -0700
committerJames Taylor <user234683@users.noreply.github.com>2019-08-11 17:09:38 -0700
commit525ec403c0d083eecae9727a0048ad9074c06e04 (patch)
treec7114cc36ea1d9537216de3f5733cefe495d588f /youtube
parent2617f87e17b94646eb5a248360a7e0f9b7675d3e (diff)
downloadyt-local-525ec403c0d083eecae9727a0048ad9074c06e04.tar.lz
yt-local-525ec403c0d083eecae9727a0048ad9074c06e04.tar.xz
yt-local-525ec403c0d083eecae9727a0048ad9074c06e04.zip
Download and save subscription thumbnails when requested by the browser, not preemptively
Diffstat (limited to 'youtube')
-rw-r--r--youtube/subscriptions.py67
1 files changed, 32 insertions, 35 deletions
diff --git a/youtube/subscriptions.py b/youtube/subscriptions.py
index c347e76..b29bf87 100644
--- a/youtube/subscriptions.py
+++ b/youtube/subscriptions.py
@@ -10,6 +10,7 @@ import json
import traceback
import contextlib
import defusedxml.ElementTree
+import urllib
import flask
from flask import request
@@ -205,31 +206,12 @@ except FileNotFoundError:
existing_thumbnails = set()
-thumbnails_queue = util.RateLimitedQueue()
check_channels_queue = util.RateLimitedQueue()
-
-
-# Use this to mark a thumbnail acceptable to be retrieved at the request of the browser
-# can't simply check if it's in the queue because items are removed when the download starts, not when it finishes
-downloading_thumbnails = set()
-
checking_channels = set()
# Just to use for printing channel checking status to console without opening database
channel_names = dict()
-def download_thumbnail_worker():
- while True:
- video_id = thumbnails_queue.get()
- try:
- success = util.download_thumbnail(thumbnails_directory, video_id)
- if success:
- existing_thumbnails.add(video_id)
- except Exception:
- traceback.print_exc()
- finally:
- downloading_thumbnails.remove(video_id)
-
def check_channel_worker():
while True:
channel_id = check_channels_queue.get()
@@ -239,20 +221,12 @@ def check_channel_worker():
checking_channels.remove(channel_id)
for i in range(0,5):
- gevent.spawn(download_thumbnail_worker)
gevent.spawn(check_channel_worker)
-
-def download_thumbnails_if_necessary(thumbnails):
- for video_id in thumbnails:
- if video_id not in existing_thumbnails and video_id not in downloading_thumbnails:
- downloading_thumbnails.add(video_id)
- thumbnails_queue.put(video_id)
-
def check_channels_if_necessary(channel_ids):
for channel_id in channel_ids:
if channel_id not in checking_channels:
@@ -279,8 +253,6 @@ def _get_upstream_videos(channel_id):
print(video_item)
videos.append((channel_id, video_item['id'], video_item['title'], video_item['duration'], video_item['time_published'], video_item['description']))
- now = time.time()
- download_thumbnails_if_necessary(video[1] for video in videos if (now - video[4]) < 30*24*3600) # Don't download thumbnails from videos older than a month
with open_database() as connection:
with connection as cursor:
@@ -488,10 +460,7 @@ def get_subscriptions_page():
with connection as cursor:
videos = []
for video in _get_videos(cursor, 60, 0):
- if video['id'] in downloading_thumbnails:
- video['thumbnail'] = util.get_thumbnail_url(video['id'])
- else:
- video['thumbnail'] = util.URL_ORIGIN + '/data/subscription_thumbnails/' + video['id'] + '.jpg'
+ video['thumbnail'] = util.URL_ORIGIN + '/data/subscription_thumbnails/' + video['id'] + '.jpg'
video['type'] = 'video'
video['item_size'] = 'small'
videos.append(video)
@@ -544,6 +513,34 @@ def post_subscriptions_page():
@yt_app.route('/data/subscription_thumbnails/<thumbnail>')
def serve_subscription_thumbnail(thumbnail):
- # .. is necessary because flask always uses the application directory at ./youtube, not the working directory
- return flask.send_from_directory(os.path.join('..', thumbnails_directory), thumbnail)
+ assert thumbnail[-4:] == '.jpg'
+ video_id = thumbnail[0:-4]
+ thumbnail_path = os.path.join(thumbnails_directory, thumbnail)
+
+ if video_id in existing_thumbnails:
+ # .. is necessary because flask always uses the application directory at ./youtube, not the working directory
+ return flask.send_from_directory(os.path.join('..', thumbnails_directory), thumbnail)
+ else:
+ url = "https://i.ytimg.com/vi/" + video_id + "/mqdefault.jpg"
+ try:
+ image = util.fetch_url(url, report_text="Saved thumbnail: " + video_id)
+ except urllib.error.HTTPError as e:
+ print("Failed to download thumbnail for " + video_id + ": " + str(e))
+ abort(e.code)
+ try:
+ f = open(thumbnail_path, 'wb')
+ except FileNotFoundError:
+ os.makedirs(thumbnail_path, exist_ok = True)
+ f = open(thumbnail_path, 'wb')
+ f.write(image)
+ f.close()
+ existing_thumbnails.add(video_id)
+
+ return flask.Response(image, mimetype='image/jpeg')
+
+
+
+
+
+