from youtube import util, yt_data_extract, channel from youtube import yt_app import settings import sqlite3 import os import time import gevent import json import traceback import contextlib import defusedxml.ElementTree import urllib import math import secrets import collections import calendar # bullshit! https://bugs.python.org/issue6280 import flask from flask import request thumbnails_directory = os.path.join(settings.data_dir, "subscription_thumbnails") # https://stackabuse.com/a-sqlite-tutorial-with-python/ database_path = os.path.join(settings.data_dir, "subscriptions.sqlite") def open_database(): if not os.path.exists(settings.data_dir): os.makedirs(settings.data_dir) connection = sqlite3.connect(database_path, check_same_thread=False) try: cursor = connection.cursor() cursor.execute('''PRAGMA foreign_keys = 1''') # Create tables if they don't exist cursor.execute('''CREATE TABLE IF NOT EXISTS subscribed_channels ( id integer PRIMARY KEY, yt_channel_id text UNIQUE NOT NULL, channel_name text NOT NULL, time_last_checked integer DEFAULT 0, next_check_time integer DEFAULT 0, muted integer DEFAULT 0 )''') cursor.execute('''CREATE TABLE IF NOT EXISTS videos ( id integer PRIMARY KEY, sql_channel_id integer NOT NULL REFERENCES subscribed_channels(id) ON UPDATE CASCADE ON DELETE CASCADE, video_id text UNIQUE NOT NULL, title text NOT NULL, duration text, time_published integer NOT NULL, is_time_published_exact integer DEFAULT 0, time_noticed integer NOT NULL, description text, watched integer default 0 )''') cursor.execute('''CREATE TABLE IF NOT EXISTS tag_associations ( id integer PRIMARY KEY, tag text NOT NULL, sql_channel_id integer NOT NULL REFERENCES subscribed_channels(id) ON UPDATE CASCADE ON DELETE CASCADE, UNIQUE(tag, sql_channel_id) )''') cursor.execute('''CREATE TABLE IF NOT EXISTS db_info ( version integer DEFAULT 1 )''') connection.commit() except: connection.rollback() connection.close() raise # https://stackoverflow.com/questions/19522505/using-sqlite3-in-python-with-with-keyword return contextlib.closing(connection) def with_open_db(function, *args, **kwargs): with open_database() as connection: with connection as cursor: return function(cursor, *args, **kwargs) def is_subscribed(channel_id): if not os.path.exists(database_path): return False with open_database() as connection: with connection as cursor: result = cursor.execute('''SELECT EXISTS( SELECT 1 FROM subscribed_channels WHERE yt_channel_id=? LIMIT 1 )''', [channel_id]).fetchone() return bool(result[0]) def _subscribe(cursor, channels): ''' channels is a list of (channel_id, channel_name) ''' channels = ( (channel_id, channel_name, 0, 0) for channel_id, channel_name in channels) cursor.executemany('''INSERT OR IGNORE INTO subscribed_channels (yt_channel_id, channel_name, time_last_checked, next_check_time) VALUES (?, ?, ?, ?)''', channels) def delete_thumbnails(to_delete): for thumbnail in to_delete: try: video_id = thumbnail[0:-4] if video_id in existing_thumbnails: os.remove(os.path.join(thumbnails_directory, thumbnail)) existing_thumbnails.remove(video_id) except Exception: print('Failed to delete thumbnail: ' + thumbnail) traceback.print_exc() def _unsubscribe(cursor, channel_ids): ''' channel_ids is a list of channel_ids ''' to_delete = [] for channel_id in channel_ids: rows = cursor.execute('''SELECT video_id FROM videos WHERE sql_channel_id = ( SELECT id FROM subscribed_channels WHERE yt_channel_id=? )''', (channel_id,)).fetchall() to_delete += [row[0] + '.jpg' for row in rows] gevent.spawn(delete_thumbnails, to_delete) cursor.executemany("DELETE FROM subscribed_channels WHERE yt_channel_id=?", ((channel_id, ) for channel_id in channel_ids)) def _get_videos(cursor, number_per_page, offset, tag = None): '''Returns a full page of videos with an offset, and a value good enough to be used as the total number of videos''' # We ask for the next 9 pages from the database # Then the actual length of the results tell us if there are more than 9 pages left, and if not, how many there actually are # This is done since there are only 9 page buttons on display at a time # If there are more than 9 pages left, we give a fake value in place of the real number of results if the entire database was queried without limit # This fake value is sufficient to get the page button generation macro to display 9 page buttons # If we wish to display more buttons this logic must change # We cannot use tricks with the sql id for the video since we frequently have filters and other restrictions in place on the results anyway # TODO: This is probably not the ideal solution if tag is not None: db_videos = cursor.execute('''SELECT video_id, title, duration, time_published, is_time_published_exact, channel_name FROM videos INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id INNER JOIN tag_associations on videos.sql_channel_id = tag_associations.sql_channel_id WHERE tag = ? AND muted = 0 ORDER BY time_noticed DESC, time_published DESC LIMIT ? OFFSET ?''', (tag, number_per_page*9, offset)).fetchall() else: db_videos = cursor.execute('''SELECT video_id, title, duration, time_published, is_time_published_exact, channel_name FROM videos INNER JOIN subscribed_channels on videos.sql_channel_id = subscribed_channels.id WHERE muted = 0 ORDER BY time_noticed DESC, time_published DESC LIMIT ? OFFSET ?''', (number_per_page*9, offset)).fetchall() pseudo_number_of_videos = offset + len(db_videos) videos = [] for db_video in db_videos[0:number_per_page]: videos.append({ 'id': db_video[0], 'title': db_video[1], 'duration': db_video[2], 'published': exact_timestamp(db_video[3]) if db_video[4] else posix_to_dumbed_down(db_video[3]), 'author': db_video[5], }) return videos, pseudo_number_of_videos def _get_subscribed_channels(cursor): for item in cursor.execute('''SELECT channel_name, yt_channel_id, muted FROM subscribed_channels ORDER BY channel_name COLLATE NOCASE'''): yield item def _add_tags(cursor, channel_ids, tags): pairs = [(tag, yt_channel_id) for tag in tags for yt_channel_id in channel_ids] cursor.executemany('''INSERT OR IGNORE INTO tag_associations (tag, sql_channel_id) SELECT ?, id FROM subscribed_channels WHERE yt_channel_id = ? ''', pairs) def _remove_tags(cursor, channel_ids, tags): pairs = [(tag, yt_channel_id) for tag in tags for yt_channel_id in channel_ids] cursor.executemany('''DELETE FROM tag_associations WHERE tag = ? AND sql_channel_id = ( SELECT id FROM subscribed_channels WHERE yt_channel_id = ? )''', pairs) def _get_tags(cursor, channel_id): return [row[0] for row in cursor.execute('''SELECT tag FROM tag_associations WHERE sql_channel_id = ( SELECT id FROM subscribed_channels WHERE yt_channel_id = ? )''', (channel_id,))] def _get_all_tags(cursor): return [row[0] for row in cursor.execute('''SELECT DISTINCT tag FROM tag_associations''')] def _get_channel_names(cursor, channel_ids): ''' returns list of (channel_id, channel_name) ''' result = [] for channel_id in channel_ids: row = cursor.execute('''SELECT channel_name FROM subscribed_channels WHERE yt_channel_id = ?''', (channel_id,)).fetchone() result.append( (channel_id, row[0]) ) return result def _channels_with_tag(cursor, tag, order=False, exclude_muted=False, include_muted_status=False): ''' returns list of (channel_id, channel_name) ''' statement = '''SELECT yt_channel_id, channel_name''' if include_muted_status: statement += ''', muted''' statement += ''' FROM subscribed_channels WHERE subscribed_channels.id IN ( SELECT tag_associations.sql_channel_id FROM tag_associations WHERE tag=? ) ''' if exclude_muted: statement += '''AND muted != 1\n''' if order: statement += '''ORDER BY channel_name COLLATE NOCASE''' return cursor.execute(statement, [tag]).fetchall() def _schedule_checking(cursor, channel_id, next_check_time): cursor.execute('''UPDATE subscribed_channels SET next_check_time = ? WHERE yt_channel_id = ?''', [int(next_check_time), channel_id]) def _is_muted(cursor, channel_id): return bool(cursor.execute('''SELECT muted FROM subscribed_channels WHERE yt_channel_id=?''', [channel_id]).fetchone()[0]) units = collections.OrderedDict([ ('year', 31536000), # 365*24*3600 ('month', 2592000), # 30*24*3600 ('week', 604800), # 7*24*3600 ('day', 86400), # 24*3600 ('hour', 3600), ('minute', 60), ('second', 1), ]) def youtube_timestamp_to_posix(dumb_timestamp): ''' Given a dumbed down timestamp such as 1 year ago, 3 hours ago, approximates the unix time (seconds since 1/1/1970) ''' dumb_timestamp = dumb_timestamp.lower() now = time.time() if dumb_timestamp == "just now": return now split = dumb_timestamp.split(' ') quantifier, unit = int(split[0]), split[1] if quantifier > 1: unit = unit[:-1] # remove s from end return now - quantifier*units[unit] def posix_to_dumbed_down(posix_time): '''Inverse of youtube_timestamp_to_posix.''' delta = int(time.time() - posix_time) assert delta >= 0 if delta == 0: return '0 seconds ago' for unit_name, unit_time in units.items(): if delta >= unit_time: quantifier = round(delta/unit_time) if quantifier == 1: return '1 ' + unit_name + ' ago' else: return str(quantifier) + ' ' + unit_name + 's ago' else: raise Exception() def exact_timestamp(posix_time): result = time.strftime('%I:%M %p %m/%d/%y', time.localtime(posix_time)) if result[0] == '0': # remove 0 infront of hour (like 01:00 PM) return result[1:] return result try: existing_thumbnails = set(os.path.splitext(name)[0] for name in os.listdir(thumbnails_directory)) except FileNotFoundError: existing_thumbnails = set() # --- Manual checking system. Rate limited in order to support very large numbers of channels to be checked --- # Auto checking system plugs into this for convenience, though it doesn't really need the rate limiting check_channels_queue = util.RateLimitedQueue() checking_channels = set() # Just to use for printing channel checking status to console without opening database channel_names = dict() def check_channel_worker(): while True: channel_id = check_channels_queue.get() try: _get_upstream_videos(channel_id) finally: checking_channels.remove(channel_id) for i in range(0,5): gevent.spawn(check_channel_worker) # ---------------------------- # --- Auto checking system - Spaghetti code --- if settings.autocheck_subscriptions: # job application format: dict with keys (channel_id, channel_name, next_check_time) autocheck_job_application = gevent.queue.Queue() # only really meant to hold 1 item, just reusing gevent's wait and timeout machinery autocheck_jobs = [] # list of dicts with the keys (channel_id, channel_name, next_check_time). Stores all the channels that need to be autochecked and when to check them with open_database() as connection: with connection as cursor: now = time.time() for row in cursor.execute('''SELECT yt_channel_id, channel_name, next_check_time FROM subscribed_channels WHERE muted != 1''').fetchall(): if row[2] is None: next_check_time = 0 else: next_check_time = row[2] # expired, check randomly within the next hour # note: even if it isn't scheduled in the past right now, it might end up being if it's due soon and we dont start dispatching by then, see below where time_until_earliest_job is negative if next_check_time < now: next_check_time = now + 3600*secrets.randbelow(60)/60 row = (row[0], row[1], next_check_time) _schedule_checking(cursor, row[0], next_check_time) autocheck_jobs.append({'channel_id': row[0], 'channel_name': row[1], 'next_check_time': next_check_time}) def autocheck_dispatcher(): '''Scans the auto_check_list. Sleeps until the earliest job is due, then adds that channel to the checking queue above. Can be sent a new job through autocheck_job_application''' while True: if len(autocheck_jobs) == 0: new_job = autocheck_job_application.get() autocheck_jobs.append(new_job) else: earliest_job_index = min(range(0, len(autocheck_jobs)), key=lambda index: autocheck_jobs[index]['next_check_time']) # https://stackoverflow.com/a/11825864 earliest_job = autocheck_jobs[earliest_job_index] time_until_earliest_job = earliest_job['next_check_time'] - time.time() if time_until_earliest_job <= -5: # should not happen unless we're running extremely slow print('ERROR: autocheck_dispatcher got job scheduled in the past, skipping and rescheduling: ' + earliest_job['channel_id'] + ', ' + earliest_job['channel_name'] + ', ' + str(earliest_job['next_check_time'])) next_check_time = time.time() + 3600*secrets.randbelow(60)/60 with_open_db(_schedule_checking, earliest_job['channel_id'], next_check_time) autocheck_jobs[earliest_job_index]['next_check_time'] = next_check_time continue # make sure it's not muted if with_open_db(_is_muted, earliest_job['channel_id']): del autocheck_jobs[earliest_job_index] continue if time_until_earliest_job > 0: # it can become less than zero (in the past) when it's set to go off while the dispatcher is doing something else at that moment try: new_job = autocheck_job_application.get(timeout = time_until_earliest_job) # sleep for time_until_earliest_job time, but allow to be interrupted by new jobs except gevent.queue.Empty: # no new jobs pass else: # new job, add it to the list autocheck_jobs.append(new_job) continue # no new jobs, time to execute the earliest job channel_names[earliest_job['channel_id']] = earliest_job['channel_name'] checking_channels.add(earliest_job['channel_id']) check_channels_queue.put(earliest_job['channel_id']) del autocheck_jobs[earliest_job_index] gevent.spawn(autocheck_dispatcher) # ---------------------------- def check_channels_if_necessary(channel_ids): for channel_id in channel_ids: if channel_id not in checking_channels: checking_channels.add(channel_id) check_channels_queue.put(channel_id) def _get_upstream_videos(channel_id): try: channel_status_name = channel_names[channel_id] except KeyError: channel_status_name = channel_id print("Checking channel: " + channel_status_name) tasks = ( gevent.spawn(channel.get_channel_tab, channel_id, print_status=False), # channel page, need for video duration gevent.spawn(util.fetch_url, 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id) # atoma feed, need for exact published time ) gevent.joinall(tasks) channel_tab, feed = tasks[0].value, tasks[1].value # extract published times from atoma feed times_published = {} try: def remove_bullshit(tag): '''Remove XML namespace bullshit from tagname. https://bugs.python.org/issue18304''' if '}' in tag: return tag[tag.rfind('}')+1:] return tag def find_element(base, tag_name): for element in base: if remove_bullshit(element.tag) == tag_name: return element return None root = defusedxml.ElementTree.fromstring(feed.decode('utf-8')) assert remove_bullshit(root.tag) == 'feed' for entry in root: if (remove_bullshit(entry.tag) != 'entry'): continue # it's yt:videoId in the xml but the yt: is turned into a namespace which is removed by remove_bullshit video_id_element = find_element(entry, 'videoId') time_published_element = find_element(entry, 'published') assert video_id_element is not None assert time_published_element is not None time_published = int(calendar.timegm(time.strptime(time_published_element.text, '%Y-%m-%dT%H:%M:%S+00:00'))) times_published[video_id_element.text] = time_published except (AssertionError, defusedxml.ElementTree.ParseError) as e: print('Failed to read atoma feed for ' + channel_status_name) traceback.print_exc() with open_database() as connection: with connection as cursor: is_first_check = cursor.execute('''SELECT time_last_checked FROM subscribed_channels WHERE yt_channel_id=?''', [channel_id]).fetchone()[0] in (None, 0) video_add_time = int(time.time()) videos = [] channel_videos = channel.extract_info(json.loads(channel_tab), 'videos')['items'] for i, video_item in enumerate(channel_videos): if 'description' not in video_item: video_item['description'] = '' if video_item['id'] in times_published: time_published = times_published[video_item['id']] is_time_published_exact = True else: is_time_published_exact = False try: time_published = youtube_timestamp_to_posix(video_item['published']) - i # subtract a few seconds off the videos so they will be in the right order except KeyError: print(video_item) if is_first_check: time_noticed = time_published # don't want a crazy ordering on first check, since we're ordering by time_noticed else: time_noticed = video_add_time videos.append((channel_id, video_item['id'], video_item['title'], video_item['duration'], time_published, is_time_published_exact, time_noticed, video_item['description'])) if len(videos) == 0: average_upload_period = 4*7*24*3600 # assume 1 month for channel with no videos elif len(videos) < 5: average_upload_period = int((time.time() - videos[len(videos)-1][4])/len(videos)) else: average_upload_period = int((time.time() - videos[4][4])/5) # equivalent to averaging the time between videos for the last 5 videos # calculate when to check next for auto checking # add some quantization and randomness to make pattern analysis by Youtube slightly harder quantized_upload_period = average_upload_period - (average_upload_period % (4*3600)) + 4*3600 # round up to nearest 4 hours randomized_upload_period = quantized_upload_period*(1 + secrets.randbelow(50)/50*0.5) # randomly between 1x and 1.5x next_check_delay = randomized_upload_period/10 # check at 10x the channel posting rate. might want to fine tune this number next_check_time = int(time.time() + next_check_delay) # calculate how many new videos there are row = cursor.execute('''SELECT video_id FROM videos INNER JOIN subscribed_channels ON videos.sql_channel_id = subscribed_channels.id WHERE yt_channel_id=? ORDER BY time_published DESC LIMIT 1''', [channel_id]).fetchone() if row is None: number_of_new_videos = len(videos) else: latest_video_id = row[0] index = 0 for video in videos: if video[1] == latest_video_id: break index += 1 number_of_new_videos = index cursor.executemany('''INSERT OR IGNORE INTO videos (sql_channel_id, video_id, title, duration, time_published, is_time_published_exact, time_noticed, description) VALUES ((SELECT id FROM subscribed_channels WHERE yt_channel_id=?), ?, ?, ?, ?, ?, ?, ?)''', videos) cursor.execute('''UPDATE subscribed_channels SET time_last_checked = ?, next_check_time = ? WHERE yt_channel_id=?''', [int(time.time()), next_check_time, channel_id]) if settings.autocheck_subscriptions: if not _is_muted(cursor, channel_id): autocheck_job_application.put({'channel_id': channel_id, 'channel_name': channel_names[channel_id], 'next_check_time': next_check_time}) if number_of_new_videos == 0: print('No new videos from ' + channel_status_name) elif number_of_new_videos == 1: print('1 new video from ' + channel_status_name) else: print(str(number_of_new_videos) + ' new videos from ' + channel_status_name) def check_all_channels(): with open_database() as connection: with connection as cursor: channel_id_name_list = cursor.execute('''SELECT yt_channel_id, channel_name FROM subscribed_channels WHERE muted != 1''').fetchall() channel_names.update(channel_id_name_list) check_channels_if_necessary([item[0] for item in channel_id_name_list]) def check_tags(tags): channel_id_name_list = [] with open_database() as connection: with connection as cursor: for tag in tags: channel_id_name_list += _channels_with_tag(cursor, tag, exclude_muted=True) channel_names.update(channel_id_name_list) check_channels_if_necessary([item[0] for item in channel_id_name_list]) def check_specific_channels(channel_ids): with open_database() as connection: with connection as cursor: channel_id_name_list = [] for channel_id in channel_ids: channel_id_name_list += cursor.execute('''SELECT yt_channel_id, channel_name FROM subscribed_channels WHERE yt_channel_id=?''', [channel_id]).fetchall() channel_names.update(channel_id_name_list) check_channels_if_necessary(channel_ids) @yt_app.route('/import_subscriptions', methods=['POST']) def import_subscriptions(): # check if the post request has the file part if 'subscriptions_file' not in request.files: #flash('No file part') return flask.redirect(util.URL_ORIGIN + request.full_path) file = request.files['subscriptions_file'] # if user does not select file, browser also # submit an empty part without filename if file.filename == '': #flash('No selected file') return flask.redirect(util.URL_ORIGIN + request.full_path) mime_type = file.mimetype if mime_type == 'application/json': file = file.read().decode('utf-8') try: file = json.loads(file) except json.decoder.JSONDecodeError: traceback.print_exc() return '400 Bad Request: Invalid json file', 400 try: channels = ( (item['snippet']['resourceId']['channelId'], item['snippet']['title']) for item in file) except (KeyError, IndexError): traceback.print_exc() return '400 Bad Request: Unknown json structure', 400 elif mime_type in ('application/xml', 'text/xml', 'text/x-opml'): file = file.read().decode('utf-8') try: root = defusedxml.ElementTree.fromstring(file) assert root.tag == 'opml' channels = [] for outline_element in root[0][0]: if (outline_element.tag != 'outline') or ('xmlUrl' not in outline_element.attrib): continue channel_name = outline_element.attrib['text'] channel_rss_url = outline_element.attrib['xmlUrl'] channel_id = channel_rss_url[channel_rss_url.find('channel_id=')+11:].strip() channels.append( (channel_id, channel_name) ) except (AssertionError, IndexError, defusedxml.ElementTree.ParseError) as e: return '400 Bad Request: Unable to read opml xml file, or the file is not the expected format', 400 else: return '400 Bad Request: Unsupported file format: ' + mime_type + '. Only subscription.json files (from Google Takeouts) and XML OPML files exported from Youtube\'s subscription manager page are supported', 400 with_open_db(_subscribe, channels) return flask.redirect(util.URL_ORIGIN + '/subscription_manager', 303) @yt_app.route('/subscription_manager', methods=['GET']) def get_subscription_manager_page(): group_by_tags = request.args.get('group_by_tags', '0') == '1' with open_database() as connection: with connection as cursor: if group_by_tags: tag_groups = [] for tag in _get_all_tags(cursor): sub_list = [] for channel_id, channel_name, muted in _channels_with_tag(cursor, tag, order=True, include_muted_status=True): sub_list.append({ 'channel_url': util.URL_ORIGIN + '/channel/' + channel_id, 'channel_name': channel_name, 'channel_id': channel_id, 'muted': muted, 'tags': [t for t in _get_tags(cursor, channel_id) if t != tag], }) tag_groups.append( (tag, sub_list) ) # Channels with no tags channel_list = cursor.execute('''SELECT yt_channel_id, channel_name, muted FROM subscribed_channels WHERE id NOT IN ( SELECT sql_channel_id FROM tag_associations ) ORDER BY channel_name COLLATE NOCASE''').fetchall() if channel_list: sub_list = [] for channel_id, channel_name, muted in channel_list: sub_list.append({ 'channel_url': util.URL_ORIGIN + '/channel/' + channel_id, 'channel_name': channel_name, 'channel_id': channel_id, 'muted': muted, 'tags': [], }) tag_groups.append( ('No tags', sub_list) ) else: sub_list = [] for channel_name, channel_id, muted in _get_subscribed_channels(cursor): sub_list.append({ 'channel_url': util.URL_ORIGIN + '/channel/' + channel_id, 'channel_name': channel_name, 'channel_id': channel_id, 'muted': muted, 'tags': _get_tags(cursor, channel_id), }) if group_by_tags: return flask.render_template('subscription_manager.html', group_by_tags = True, tag_groups = tag_groups, ) else: return flask.render_template('subscription_manager.html', group_by_tags = False, sub_list = sub_list, ) def list_from_comma_separated_tags(string): return [tag.strip() for tag in string.split(',') if tag.strip()] @yt_app.route('/subscription_manager', methods=['POST']) def post_subscription_manager_page(): action = request.values['action'] with open_database() as connection: with connection as cursor: if action == 'add_tags': _add_tags(cursor, request.values.getlist('channel_ids'), [tag.lower() for tag in list_from_comma_separated_tags(request.values['tags'])]) elif action == 'remove_tags': _remove_tags(cursor, request.values.getlist('channel_ids'), [tag.lower() for tag in list_from_comma_separated_tags(request.values['tags'])]) elif action == 'unsubscribe': _unsubscribe(cursor, request.values.getlist('channel_ids')) elif action == 'unsubscribe_verify': unsubscribe_list = _get_channel_names(cursor, request.values.getlist('channel_ids')) return flask.render_template('unsubscribe_verify.html', unsubscribe_list = unsubscribe_list) elif action == 'mute': cursor.executemany('''UPDATE subscribed_channels SET muted = 1 WHERE yt_channel_id = ?''', [(ci,) for ci in request.values.getlist('channel_ids')]) elif action == 'unmute': cursor.executemany('''UPDATE subscribed_channels SET muted = 0 WHERE yt_channel_id = ?''', [(ci,) for ci in request.values.getlist('channel_ids')]) else: flask.abort(400) return flask.redirect(util.URL_ORIGIN + request.full_path, 303) @yt_app.route('/subscriptions', methods=['GET']) @yt_app.route('/feed/subscriptions', methods=['GET']) def get_subscriptions_page(): page = int(request.args.get('page', 1)) with open_database() as connection: with connection as cursor: tag = request.args.get('tag', None) videos, number_of_videos_in_db = _get_videos(cursor, 60, (page - 1)*60, tag) for video in videos: video['thumbnail'] = util.URL_ORIGIN + '/data/subscription_thumbnails/' + video['id'] + '.jpg' video['type'] = 'video' video['item_size'] = 'small' yt_data_extract.add_extra_html_info(video) tags = _get_all_tags(cursor) subscription_list = [] for channel_name, channel_id, muted in _get_subscribed_channels(cursor): subscription_list.append({ 'channel_url': util.URL_ORIGIN + '/channel/' + channel_id, 'channel_name': channel_name, 'channel_id': channel_id, 'muted': muted, }) return flask.render_template('subscriptions.html', videos = videos, num_pages = math.ceil(number_of_videos_in_db/60), parameters_dictionary = request.args, tags = tags, current_tag = tag, subscription_list = subscription_list, ) @yt_app.route('/subscriptions', methods=['POST']) @yt_app.route('/feed/subscriptions', methods=['POST']) def post_subscriptions_page(): action = request.values['action'] if action == 'subscribe': if len(request.values.getlist('channel_id')) != len(request.values.getlist('channel_name')): return '400 Bad Request, length of channel_id != length of channel_name', 400 with_open_db(_subscribe, zip(request.values.getlist('channel_id'), request.values.getlist('channel_name'))) elif action == 'unsubscribe': with_open_db(_unsubscribe, request.values.getlist('channel_id')) elif action == 'refresh': type = request.values['type'] if type == 'all': check_all_channels() elif type == 'tag': check_tags(request.values.getlist('tag_name')) elif type == 'channel': check_specific_channels(request.values.getlist('channel_id')) else: flask.abort(400) else: flask.abort(400) return '', 204 @yt_app.route('/data/subscription_thumbnails/') def serve_subscription_thumbnail(thumbnail): '''Serves thumbnail from disk if it's been saved already. If not, downloads the thumbnail, saves to disk, and serves it.''' assert thumbnail[-4:] == '.jpg' video_id = thumbnail[0:-4] thumbnail_path = os.path.join(thumbnails_directory, thumbnail) if video_id in existing_thumbnails: try: f = open(thumbnail_path, 'rb') except FileNotFoundError: existing_thumbnails.remove(video_id) else: image = f.read() f.close() return flask.Response(image, mimetype='image/jpeg') url = "https://i.ytimg.com/vi/" + video_id + "/mqdefault.jpg" try: image = util.fetch_url(url, report_text="Saved thumbnail: " + video_id) except urllib.error.HTTPError as e: print("Failed to download thumbnail for " + video_id + ": " + str(e)) abort(e.code) try: f = open(thumbnail_path, 'wb') except FileNotFoundError: os.makedirs(thumbnails_directory, exist_ok = True) f = open(thumbnail_path, 'wb') f.write(image) f.close() existing_thumbnails.add(video_id) return flask.Response(image, mimetype='image/jpeg')