From 06051dd127fd9805442ee10b569f36a611dbbc8e Mon Sep 17 00:00:00 2001 From: Astounds Date: Tue, 31 Mar 2026 21:38:51 -0500 Subject: fix: support YouTube 2024+ data formats for playlists, podcasts and channels - Add PODCAST content type support in lockupViewModel extraction - Extract thumbnails and episode count from thumbnail overlay badges - Migrate playlist page fetching from pbj=1 to innertube API (youtubei/v1/browse) - Support new pageHeaderRenderer format in playlist metadata extraction - Fix subscriber count extraction when YouTube returns handle instead of count - Hide "None subscribers" in template when data is unavailable --- youtube/yt_data_extract/everything_else.py | 112 ++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 26 deletions(-) (limited to 'youtube/yt_data_extract/everything_else.py') diff --git a/youtube/yt_data_extract/everything_else.py b/youtube/yt_data_extract/everything_else.py index 0f64649..5930111 100644 --- a/youtube/yt_data_extract/everything_else.py +++ b/youtube/yt_data_extract/everything_else.py @@ -218,39 +218,99 @@ def extract_playlist_metadata(polymer_json): return {'error': err} metadata = {'error': None} + metadata['title'] = None + metadata['first_video_id'] = None + metadata['thumbnail'] = None + metadata['video_count'] = None + metadata['description'] = '' + metadata['author'] = None + metadata['author_id'] = None + metadata['author_url'] = None + metadata['view_count'] = None + metadata['like_count'] = None + metadata['time_published'] = None + header = deep_get(response, 'header', 'playlistHeaderRenderer', default={}) - metadata['title'] = extract_str(header.get('title')) - metadata['first_video_id'] = deep_get(header, 'playEndpoint', 'watchEndpoint', 'videoId') - first_id = re.search(r'([a-z_\-]{11})', deep_get(header, - 'thumbnail', 'thumbnails', 0, 'url', default='')) - if first_id: - conservative_update(metadata, 'first_video_id', first_id.group(1)) - if metadata['first_video_id'] is None: - metadata['thumbnail'] = None + if header: + # Classic playlistHeaderRenderer format + metadata['title'] = extract_str(header.get('title')) + metadata['first_video_id'] = deep_get(header, 'playEndpoint', 'watchEndpoint', 'videoId') + first_id = re.search(r'([a-z_\-]{11})', deep_get(header, + 'thumbnail', 'thumbnails', 0, 'url', default='')) + if first_id: + conservative_update(metadata, 'first_video_id', first_id.group(1)) + + metadata['video_count'] = extract_int(header.get('numVideosText')) + metadata['description'] = extract_str(header.get('descriptionText'), default='') + metadata['author'] = extract_str(header.get('ownerText')) + metadata['author_id'] = multi_deep_get(header, + ['ownerText', 'runs', 0, 'navigationEndpoint', 'browseEndpoint', 'browseId'], + ['ownerEndpoint', 'browseEndpoint', 'browseId']) + metadata['view_count'] = extract_int(header.get('viewCountText')) + metadata['like_count'] = extract_int(header.get('likesCountWithoutLikeText')) + for stat in header.get('stats', ()): + text = extract_str(stat) + if 'videos' in text or 'episodes' in text: + conservative_update(metadata, 'video_count', extract_int(text)) + elif 'views' in text: + conservative_update(metadata, 'view_count', extract_int(text)) + elif 'updated' in text: + metadata['time_published'] = extract_date(text) else: - metadata['thumbnail'] = f"https://i.ytimg.com/vi/{metadata['first_video_id']}/hqdefault.jpg" + # New pageHeaderRenderer format (YouTube 2024+) + page_header = deep_get(response, 'header', 'pageHeaderRenderer', default={}) + metadata['title'] = page_header.get('pageTitle') + view_model = deep_get(page_header, 'content', 'pageHeaderViewModel', default={}) + + # Extract title from viewModel if not found + if not metadata['title']: + metadata['title'] = deep_get(view_model, + 'title', 'dynamicTextViewModel', 'text', 'content') + + # Extract metadata from rows (author, video count, views, etc.) + meta_rows = deep_get(view_model, + 'metadata', 'contentMetadataViewModel', 'metadataRows', default=[]) + for row in meta_rows: + for part in row.get('metadataParts', []): + text_content = deep_get(part, 'text', 'content', default='') + # Author from avatarStack + avatar_stack = deep_get(part, 'avatarStack', 'avatarStackViewModel', default={}) + if avatar_stack: + author_text = deep_get(avatar_stack, 'text', 'content') + if author_text: + metadata['author'] = author_text + # Extract author_id from commandRuns + for run in deep_get(avatar_stack, 'text', 'commandRuns', default=[]): + browse_id = deep_get(run, 'onTap', 'innertubeCommand', + 'browseEndpoint', 'browseId') + if browse_id: + metadata['author_id'] = browse_id + # Video/episode count + if text_content and ('video' in text_content.lower() or 'episode' in text_content.lower()): + conservative_update(metadata, 'video_count', extract_int(text_content)) + # View count + elif text_content and 'view' in text_content.lower(): + conservative_update(metadata, 'view_count', extract_int(text_content)) + # Last updated + elif text_content and 'updated' in text_content.lower(): + metadata['time_published'] = extract_date(text_content) + + # Extract description from sidebar if available + sidebar = deep_get(response, 'sidebar', 'playlistSidebarRenderer', 'items', default=[]) + for sidebar_item in sidebar: + desc = deep_get(sidebar_item, 'playlistSidebarPrimaryInfoRenderer', + 'description', 'simpleText') + if desc: + metadata['description'] = desc - metadata['video_count'] = extract_int(header.get('numVideosText')) - metadata['description'] = extract_str(header.get('descriptionText'), default='') - metadata['author'] = extract_str(header.get('ownerText')) - metadata['author_id'] = multi_deep_get(header, - ['ownerText', 'runs', 0, 'navigationEndpoint', 'browseEndpoint', 'browseId'], - ['ownerEndpoint', 'browseEndpoint', 'browseId']) if metadata['author_id']: metadata['author_url'] = 'https://www.youtube.com/channel/' + metadata['author_id'] + + if metadata['first_video_id'] is None: + metadata['thumbnail'] = None else: - metadata['author_url'] = None - metadata['view_count'] = extract_int(header.get('viewCountText')) - metadata['like_count'] = extract_int(header.get('likesCountWithoutLikeText')) - for stat in header.get('stats', ()): - text = extract_str(stat) - if 'videos' in text: - conservative_update(metadata, 'video_count', extract_int(text)) - elif 'views' in text: - conservative_update(metadata, 'view_count', extract_int(text)) - elif 'updated' in text: - metadata['time_published'] = extract_date(text) + metadata['thumbnail'] = f"https://i.ytimg.com/vi/{metadata['first_video_id']}/hqdefault.jpg" microformat = deep_get(response, 'microformat', 'microformatDataRenderer', default={}) -- cgit v1.2.3