diff options
Diffstat (limited to 'youtube/subscriptions.py')
| -rw-r--r-- | youtube/subscriptions.py | 121 |
1 files changed, 68 insertions, 53 deletions
diff --git a/youtube/subscriptions.py b/youtube/subscriptions.py index 6f75578..b841f5d 100644 --- a/youtube/subscriptions.py +++ b/youtube/subscriptions.py @@ -26,6 +26,7 @@ thumbnails_directory = os.path.join(settings.data_dir, "subscription_thumbnails" database_path = os.path.join(settings.data_dir, "subscriptions.sqlite") + def open_database(): if not os.path.exists(settings.data_dir): os.makedirs(settings.data_dir) @@ -74,11 +75,13 @@ def open_database(): # https://stackoverflow.com/questions/19522505/using-sqlite3-in-python-with-with-keyword return contextlib.closing(connection) + def with_open_db(function, *args, **kwargs): with open_database() as connection: with connection as cursor: return function(cursor, *args, **kwargs) + def _is_subscribed(cursor, channel_id): result = cursor.execute('''SELECT EXISTS( SELECT 1 @@ -88,12 +91,14 @@ def _is_subscribed(cursor, channel_id): )''', [channel_id]).fetchone() return bool(result[0]) + def is_subscribed(channel_id): if not os.path.exists(database_path): return False return with_open_db(_is_subscribed, channel_id) + def _subscribe(channels): ''' channels is a list of (channel_id, channel_name) ''' channels = list(channels) @@ -101,7 +106,8 @@ def _subscribe(channels): with connection as cursor: channel_ids_to_check = [channel[0] for channel in channels if not _is_subscribed(cursor, channel[0])] - rows = ( (channel_id, channel_name, 0, 0) for channel_id, channel_name in channels) + rows = ((channel_id, channel_name, 0, 0) for channel_id, + channel_name in channels) cursor.executemany('''INSERT OR IGNORE INTO subscribed_channels (yt_channel_id, channel_name, time_last_checked, next_check_time) VALUES (?, ?, ?, ?)''', rows) @@ -111,6 +117,7 @@ def _subscribe(channels): channel_names.update(channels) check_channels_if_necessary(channel_ids_to_check) + def delete_thumbnails(to_delete): for thumbnail in to_delete: try: @@ -122,6 +129,7 @@ def delete_thumbnails(to_delete): print('Failed to delete thumbnail: ' + thumbnail) traceback.print_exc() + def _unsubscribe(cursor, channel_ids): ''' channel_ids is a list of channel_ids ''' to_delete = [] @@ -138,7 +146,8 @@ def _unsubscribe(cursor, channel_ids): gevent.spawn(delete_thumbnails, to_delete) cursor.executemany("DELETE FROM subscribed_channels WHERE yt_channel_id=?", ((channel_id, ) for channel_id in channel_ids)) -def _get_videos(cursor, number_per_page, offset, tag = None): + +def _get_videos(cursor, number_per_page, offset, tag=None): '''Returns a full page of videos with an offset, and a value good enough to be used as the total number of videos''' # We ask for the next 9 pages from the database # Then the actual length of the results tell us if there are more than 9 pages left, and if not, how many there actually are @@ -181,8 +190,6 @@ def _get_videos(cursor, number_per_page, offset, tag = None): return videos, pseudo_number_of_videos - - def _get_subscribed_channels(cursor): for item in cursor.execute('''SELECT channel_name, yt_channel_id, muted FROM subscribed_channels @@ -204,7 +211,6 @@ def _remove_tags(cursor, channel_ids, tags): )''', pairs) - def _get_tags(cursor, channel_id): return [row[0] for row in cursor.execute('''SELECT tag FROM tag_associations @@ -212,9 +218,11 @@ def _get_tags(cursor, channel_id): SELECT id FROM subscribed_channels WHERE yt_channel_id = ? )''', (channel_id,))] + def _get_all_tags(cursor): return [row[0] for row in cursor.execute('''SELECT DISTINCT tag FROM tag_associations''')] + def _get_channel_names(cursor, channel_ids): ''' returns list of (channel_id, channel_name) ''' result = [] @@ -222,11 +230,12 @@ def _get_channel_names(cursor, channel_ids): row = cursor.execute('''SELECT channel_name FROM subscribed_channels WHERE yt_channel_id = ?''', (channel_id,)).fetchone() - result.append( (channel_id, row[0]) ) + result.append((channel_id, row[0])) return result -def _channels_with_tag(cursor, tag, order=False, exclude_muted=False, include_muted_status=False): +def _channels_with_tag(cursor, tag, order=False, exclude_muted=False, + include_muted_status=False): ''' returns list of (channel_id, channel_name) ''' statement = '''SELECT yt_channel_id, channel_name''' @@ -247,12 +256,15 @@ def _channels_with_tag(cursor, tag, order=False, exclude_muted=False, include_mu return cursor.execute(statement, [tag]).fetchall() + def _schedule_checking(cursor, channel_id, next_check_time): cursor.execute('''UPDATE subscribed_channels SET next_check_time = ? WHERE yt_channel_id = ?''', [int(next_check_time), channel_id]) + def _is_muted(cursor, channel_id): return bool(cursor.execute('''SELECT muted FROM subscribed_channels WHERE yt_channel_id=?''', [channel_id]).fetchone()[0]) + units = collections.OrderedDict([ ('year', 31536000), # 365*24*3600 ('month', 2592000), # 30*24*3600 @@ -262,6 +274,8 @@ units = collections.OrderedDict([ ('minute', 60), ('second', 1), ]) + + def youtube_timestamp_to_posix(dumb_timestamp): ''' Given a dumbed down timestamp such as 1 year ago, 3 hours ago, approximates the unix time (seconds since 1/1/1970) ''' @@ -275,6 +289,7 @@ def youtube_timestamp_to_posix(dumb_timestamp): unit = unit[:-1] # remove s from end return now - quantifier*units[unit] + def posix_to_dumbed_down(posix_time): '''Inverse of youtube_timestamp_to_posix.''' delta = int(time.time() - posix_time) @@ -293,12 +308,14 @@ def posix_to_dumbed_down(posix_time): else: raise Exception() + def exact_timestamp(posix_time): result = time.strftime('%I:%M %p %m/%d/%y', time.localtime(posix_time)) if result[0] == '0': # remove 0 infront of hour (like 01:00 PM) return result[1:] return result + try: existing_thumbnails = set(os.path.splitext(name)[0] for name in os.listdir(thumbnails_directory)) except FileNotFoundError: @@ -314,6 +331,7 @@ checking_channels = set() # Just to use for printing channel checking status to console without opening database channel_names = dict() + def check_channel_worker(): while True: channel_id = check_channels_queue.get() @@ -324,12 +342,12 @@ def check_channel_worker(): finally: checking_channels.remove(channel_id) -for i in range(0,5): + +for i in range(0, 5): gevent.spawn(check_channel_worker) # ---------------------------- - # --- Auto checking system - Spaghetti code --- def autocheck_dispatcher(): '''Scans the auto_check_list. Sleeps until the earliest job is due, then adds that channel to the checking queue above. Can be sent a new job through autocheck_job_application''' @@ -356,7 +374,7 @@ def autocheck_dispatcher(): if time_until_earliest_job > 0: # it can become less than zero (in the past) when it's set to go off while the dispatcher is doing something else at that moment try: - new_job = autocheck_job_application.get(timeout = time_until_earliest_job) # sleep for time_until_earliest_job time, but allow to be interrupted by new jobs + new_job = autocheck_job_application.get(timeout=time_until_earliest_job) # sleep for time_until_earliest_job time, but allow to be interrupted by new jobs except gevent.queue.Empty: # no new jobs pass else: # new job, add it to the list @@ -369,7 +387,10 @@ def autocheck_dispatcher(): check_channels_queue.put(earliest_job['channel_id']) del autocheck_jobs[earliest_job_index] + dispatcher_greenlet = None + + def start_autocheck_system(): global autocheck_job_application global autocheck_jobs @@ -398,30 +419,34 @@ def start_autocheck_system(): autocheck_jobs.append({'channel_id': row[0], 'channel_name': row[1], 'next_check_time': next_check_time}) dispatcher_greenlet = gevent.spawn(autocheck_dispatcher) + def stop_autocheck_system(): if dispatcher_greenlet is not None: dispatcher_greenlet.kill() + def autocheck_setting_changed(old_value, new_value): if new_value: start_autocheck_system() else: stop_autocheck_system() -settings.add_setting_changed_hook('autocheck_subscriptions', + +settings.add_setting_changed_hook( + 'autocheck_subscriptions', autocheck_setting_changed) if settings.autocheck_subscriptions: start_autocheck_system() # ---------------------------- - def check_channels_if_necessary(channel_ids): for channel_id in channel_ids: if channel_id not in checking_channels: checking_channels.add(channel_id) check_channels_queue.put(channel_id) + def _get_atoma_feed(channel_id): url = 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id try: @@ -432,6 +457,7 @@ def _get_atoma_feed(channel_id): return '' raise + def _get_channel_tab(channel_id, channel_status_name): try: return channel.get_channel_tab(channel_id, print_status=False) @@ -447,6 +473,7 @@ def _get_channel_tab(channel_id, channel_status_name): return None raise + def _get_upstream_videos(channel_id): try: channel_status_name = channel_names[channel_id] @@ -527,9 +554,8 @@ def _get_upstream_videos(channel_id): video_item['channel_id'] = channel_id - if len(videos) == 0: - average_upload_period = 4*7*24*3600 # assume 1 month for channel with no videos + average_upload_period = 4*7*24*3600 # assume 1 month for channel with no videos elif len(videos) < 5: average_upload_period = int((time.time() - videos[len(videos)-1]['time_published'])/len(videos)) else: @@ -591,7 +617,6 @@ def _get_upstream_videos(channel_id): video_item['description'], )) - cursor.executemany('''INSERT OR IGNORE INTO videos ( sql_channel_id, video_id, @@ -619,7 +644,6 @@ def _get_upstream_videos(channel_id): print(str(number_of_new_videos) + ' new videos from ' + channel_status_name) - def check_all_channels(): with open_database() as connection: with connection as cursor: @@ -654,22 +678,20 @@ def check_specific_channels(channel_ids): check_channels_if_necessary(channel_ids) - @yt_app.route('/import_subscriptions', methods=['POST']) def import_subscriptions(): # check if the post request has the file part if 'subscriptions_file' not in request.files: - #flash('No file part') + # flash('No file part') return flask.redirect(util.URL_ORIGIN + request.full_path) file = request.files['subscriptions_file'] # if user does not select file, browser also # submit an empty part without filename if file.filename == '': - #flash('No selected file') + # flash('No selected file') return flask.redirect(util.URL_ORIGIN + request.full_path) - mime_type = file.mimetype if mime_type == 'application/json': @@ -681,7 +703,7 @@ def import_subscriptions(): return '400 Bad Request: Invalid json file', 400 try: - channels = ( (item['snippet']['resourceId']['channelId'], item['snippet']['title']) for item in file) + channels = ((item['snippet']['resourceId']['channelId'], item['snippet']['title']) for item in file) except (KeyError, IndexError): traceback.print_exc() return '400 Bad Request: Unknown json structure', 400 @@ -695,11 +717,10 @@ def import_subscriptions(): if (outline_element.tag != 'outline') or ('xmlUrl' not in outline_element.attrib): continue - channel_name = outline_element.attrib['text'] channel_rss_url = outline_element.attrib['xmlUrl'] channel_id = channel_rss_url[channel_rss_url.find('channel_id=')+11:].strip() - channels.append( (channel_id, channel_name) ) + channels.append((channel_id, channel_name)) except (AssertionError, IndexError, defusedxml.ElementTree.ParseError) as e: return '400 Bad Request: Unable to read opml xml file, or the file is not the expected format', 400 @@ -711,7 +732,6 @@ def import_subscriptions(): return flask.redirect(util.URL_ORIGIN + '/subscription_manager', 303) - @yt_app.route('/subscription_manager', methods=['GET']) def get_subscription_manager_page(): group_by_tags = request.args.get('group_by_tags', '0') == '1' @@ -731,7 +751,7 @@ def get_subscription_manager_page(): 'tags': [t for t in _get_tags(cursor, channel_id) if t != tag], }) - tag_groups.append( (tag, sub_list) ) + tag_groups.append((tag, sub_list)) # Channels with no tags channel_list = cursor.execute('''SELECT yt_channel_id, channel_name, muted @@ -751,7 +771,7 @@ def get_subscription_manager_page(): 'tags': [], }) - tag_groups.append( ('No tags', sub_list) ) + tag_groups.append(('No tags', sub_list)) else: sub_list = [] for channel_name, channel_id, muted in _get_subscribed_channels(cursor): @@ -763,20 +783,20 @@ def get_subscription_manager_page(): 'tags': _get_tags(cursor, channel_id), }) - - - if group_by_tags: - return flask.render_template('subscription_manager.html', - group_by_tags = True, - tag_groups = tag_groups, + return flask.render_template( + 'subscription_manager.html', + group_by_tags=True, + tag_groups=tag_groups, ) else: - return flask.render_template('subscription_manager.html', - group_by_tags = False, - sub_list = sub_list, + return flask.render_template( + 'subscription_manager.html', + group_by_tags=False, + sub_list=sub_list, ) + def list_from_comma_separated_tags(string): return [tag.strip() for tag in string.split(',') if tag.strip()] @@ -795,7 +815,7 @@ def post_subscription_manager_page(): _unsubscribe(cursor, request.values.getlist('channel_ids')) elif action == 'unsubscribe_verify': unsubscribe_list = _get_channel_names(cursor, request.values.getlist('channel_ids')) - return flask.render_template('unsubscribe_verify.html', unsubscribe_list = unsubscribe_list) + return flask.render_template('unsubscribe_verify.html', unsubscribe_list=unsubscribe_list) elif action == 'mute': cursor.executemany('''UPDATE subscribed_channels @@ -810,6 +830,7 @@ def post_subscription_manager_page(): return flask.redirect(util.URL_ORIGIN + request.full_path, 303) + @yt_app.route('/subscriptions', methods=['GET']) @yt_app.route('/feed/subscriptions', methods=['GET']) def get_subscriptions_page(): @@ -826,7 +847,6 @@ def get_subscriptions_page(): tags = _get_all_tags(cursor) - subscription_list = [] for channel_name, channel_id, muted in _get_subscribed_channels(cursor): subscription_list.append({ @@ -836,16 +856,18 @@ def get_subscriptions_page(): 'muted': muted, }) - return flask.render_template('subscriptions.html', - header_playlist_names = local_playlist.get_playlist_names(), - videos = videos, - num_pages = math.ceil(number_of_videos_in_db/60), - parameters_dictionary = request.args, - tags = tags, - current_tag = tag, - subscription_list = subscription_list, + return flask.render_template( + 'subscriptions.html', + header_playlist_names=local_playlist.get_playlist_names(), + videos=videos, + num_pages=math.ceil(number_of_videos_in_db/60), + parameters_dictionary=request.args, + tags=tags, + current_tag=tag, + subscription_list=subscription_list, ) + @yt_app.route('/subscriptions', methods=['POST']) @yt_app.route('/feed/subscriptions', methods=['POST']) def post_subscriptions_page(): @@ -900,17 +922,10 @@ def serve_subscription_thumbnail(thumbnail): try: f = open(thumbnail_path, 'wb') except FileNotFoundError: - os.makedirs(thumbnails_directory, exist_ok = True) + os.makedirs(thumbnails_directory, exist_ok=True) f = open(thumbnail_path, 'wb') f.write(image) f.close() existing_thumbnails.add(video_id) return flask.Response(image, mimetype='image/jpeg') - - - - - - - |