From 3dee7ea0d1156642d02f504b9229676b287ddf0a Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 7 Aug 2021 17:05:58 -0700 Subject: Switch to new comments api now that old one is being disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit watch_comment api periodically gives the error "Top level comments mweb servlet is turned down." The continuation items for the new api are in a different arrangement in the json, so changes were necessary to the extract_items function. Signed-off-by: Jesús --- youtube/yt_data_extract/common.py | 39 ++++++++++++++++++++++++------ youtube/yt_data_extract/everything_else.py | 6 ++--- 2 files changed, 34 insertions(+), 11 deletions(-) (limited to 'youtube/yt_data_extract') diff --git a/youtube/yt_data_extract/common.py b/youtube/yt_data_extract/common.py index e87808b..74ac1d6 100644 --- a/youtube/yt_data_extract/common.py +++ b/youtube/yt_data_extract/common.py @@ -478,6 +478,22 @@ def extract_items_from_renderer(renderer, item_types=_item_types): renderer = None + +def extract_items_from_renderer_list(renderers, item_types=_item_types): + '''Same as extract_items_from_renderer, but provide a list of renderers''' + items = [] + ctoken = None + for renderer in renderers: + new_items, new_ctoken = extract_items_from_renderer( + renderer, + item_types=item_types) + items += new_items + # prioritize ctoken associated with items + if (not ctoken) or (new_ctoken and new_items): + ctoken = new_ctoken + return items, ctoken + + def extract_items(response, item_types=_item_types, search_engagement_panels=False): '''return items, ctoken''' @@ -495,6 +511,15 @@ def extract_items(response, item_types=_item_types, item_types=item_types) if items: break + elif 'onResponseReceivedEndpoints' in response: + for endpoint in response.get('onResponseReceivedEndpoints', []): + items, ctoken = extract_items_from_renderer_list( + deep_get(endpoint, 'appendContinuationItemsAction', + 'continuationItems', default=[]), + item_types=item_types, + ) + if items: + break elif 'contents' in response: renderer = get(response, 'contents', {}) items, ctoken = extract_items_from_renderer( @@ -502,11 +527,11 @@ def extract_items(response, item_types=_item_types, item_types=item_types) if search_engagement_panels and 'engagementPanels' in response: - for engagement_renderer in response['engagementPanels']: - additional_items, cont = extract_items_from_renderer( - engagement_renderer, - item_types=item_types) - items += additional_items - if cont and not ctoken: - ctoken = cont + new_items, new_ctoken = extract_items_from_renderer_list( + response['engagementPanels'], item_types=item_types + ) + items += new_items + if (not ctoken) or (new_ctoken and new_items): + ctoken = new_ctoken + return items, ctoken diff --git a/youtube/yt_data_extract/everything_else.py b/youtube/yt_data_extract/everything_else.py index 7275975..ba3cd77 100644 --- a/youtube/yt_data_extract/everything_else.py +++ b/youtube/yt_data_extract/everything_else.py @@ -222,15 +222,13 @@ def _ctoken_metadata(ctoken): result['sort'] = 0 return result -def extract_comments_info(polymer_json): +def extract_comments_info(polymer_json, ctoken=None): response, err = extract_response(polymer_json) if err: return {'error': err} info = {'error': None} - url = multi_deep_get(polymer_json, [1, 'url'], ['url']) - if url: - ctoken = urllib.parse.parse_qs(url[url.find('?')+1:])['ctoken'][0] + if ctoken: metadata = _ctoken_metadata(ctoken) else: metadata = {} -- cgit v1.2.3