From f787e4e2027583476ca34bd01c8462f6459369bb Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 31 Jan 2020 20:06:15 -0800 Subject: Give a proper error message for 429 errors These occur when too many requests are coming from a Tor exit node. Before, there would be an error page with an exception instructing users to report the issue. But this is an expected and persistent issue. --- youtube/subscriptions.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'youtube/subscriptions.py') diff --git a/youtube/subscriptions.py b/youtube/subscriptions.py index 76130f3..c26c79d 100644 --- a/youtube/subscriptions.py +++ b/youtube/subscriptions.py @@ -405,7 +405,14 @@ def check_channels_if_necessary(channel_ids): checking_channels.add(channel_id) check_channels_queue.put(channel_id) - +def _get_atoma_feed(channel_id): + url = 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id + try: + return util.fetch_url(url).decode('utf-8') + except util.FetchError as e: + if e.code == '404': # 404 is expected for terminated channels + return '' + raise def _get_upstream_videos(channel_id): try: @@ -417,7 +424,7 @@ def _get_upstream_videos(channel_id): tasks = ( gevent.spawn(channel.get_channel_tab, channel_id, print_status=False), # channel page, need for video duration - gevent.spawn(util.fetch_url, 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id) # atoma feed, need for exact published time + gevent.spawn(_get_atoma_feed, channel_id) # need atoma feed for exact published time ) gevent.joinall(tasks) @@ -438,7 +445,7 @@ def _get_upstream_videos(channel_id): return element return None - root = defusedxml.ElementTree.fromstring(feed.decode('utf-8')) + root = defusedxml.ElementTree.fromstring(feed) assert remove_bullshit(root.tag) == 'feed' for entry in root: if (remove_bullshit(entry.tag) != 'entry'): -- cgit v1.2.3