diff options
Diffstat (limited to 'youtube_dlc/extractor/bandcamp.py')
| -rw-r--r-- | youtube_dlc/extractor/bandcamp.py | 268 |
1 files changed, 111 insertions, 157 deletions
diff --git a/youtube_dlc/extractor/bandcamp.py b/youtube_dlc/extractor/bandcamp.py index 0e7492764..69e673a26 100644 --- a/youtube_dlc/extractor/bandcamp.py +++ b/youtube_dlc/extractor/bandcamp.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals import random @@ -5,10 +6,7 @@ import re import time from .common import InfoExtractor -from ..compat import ( - compat_str, - compat_urlparse, -) +from ..compat import compat_str from ..utils import ( ExtractorError, float_or_none, @@ -17,71 +15,32 @@ from ..utils import ( parse_filesize, str_or_none, try_get, - unescapeHTML, update_url_query, unified_strdate, unified_timestamp, url_or_none, + urljoin, ) -class BandcampBaseIE(InfoExtractor): - """Provide base functions for Bandcamp extractors""" - - def _extract_json_from_html_data_attribute(self, webpage, suffix, video_id): - json_string = self._html_search_regex( - r' data-%s="([^"]*)' % suffix, - webpage, '%s json' % suffix, default='{}') - - return self._parse_json(json_string, video_id) - - def _parse_json_track(self, json): - formats = [] - file_ = json.get('file') - if isinstance(file_, dict): - for format_id, format_url in file_.items(): - if not url_or_none(format_url): - continue - ext, abr_str = format_id.split('-', 1) - formats.append({ - 'format_id': format_id, - 'url': self._proto_relative_url(format_url, 'http:'), - 'ext': ext, - 'vcodec': 'none', - 'acodec': ext, - 'abr': int_or_none(abr_str), - }) - - return { - 'duration': float_or_none(json.get('duration')), - 'id': str_or_none(json.get('track_id') or json.get('id')), - 'title': json.get('title'), - 'title_link': json.get('title_link'), - 'number': int_or_none(json.get('track_num')), - 'formats': formats - } - - -class BandcampIE(BandcampBaseIE): - IE_NAME = "Bandcamp:track" - _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<title>[^/?#&]+)' +class BandcampIE(InfoExtractor): + _VALID_URL = r'https?://[^/]+\.bandcamp\.com/track/(?P<id>[^/?#&]+)' _TESTS = [{ 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', 'md5': 'c557841d5e50261777a6585648adf439', 'info_dict': { 'id': '1812978515', 'ext': 'mp3', - 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad", + 'title': "youtube-dl \"'/\\ä↭ - youtube-dl \"'/\\ä↭ - youtube-dl test song \"'/\\ä↭", 'duration': 9.8485, - 'uploader': "youtube-dl \"'/\\\u00e4\u21ad", - 'timestamp': 1354224127, + 'uploader': 'youtube-dl "\'/\\ä↭', 'upload_date': '20121129', + 'timestamp': 1354224127, }, '_skip': 'There is a limit of 200 free downloads / month for the test song' }, { # free download 'url': 'http://benprunty.bandcamp.com/track/lanius-battle', - 'md5': '5d92af55811e47f38962a54c30b07ef0', 'info_dict': { 'id': '2650410135', 'ext': 'aiff', @@ -120,52 +79,59 @@ class BandcampIE(BandcampBaseIE): }, }] + def _extract_data_attr(self, webpage, video_id, attr='tralbum', fatal=True): + return self._parse_json(self._html_search_regex( + r'data-%s=(["\'])({.+?})\1' % attr, webpage, + attr + ' data', group=2), video_id, fatal=fatal) + def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - title = mobj.group('title') - url_track_title = title + title = self._match_id(url) webpage = self._download_webpage(url, title) - thumbnail = self._html_search_meta('og:image', webpage, default=None) - - json_tralbum = self._extract_json_from_html_data_attribute(webpage, "tralbum", url_track_title) - json_embed = self._extract_json_from_html_data_attribute(webpage, "embed", url_track_title) - - json_tracks = json_tralbum.get('trackinfo') - if not json_tracks: - raise ExtractorError('Could not extract track') - - track = self._parse_json_track(json_tracks[0]) - artist = json_tralbum.get('artist') - album_title = json_embed.get('album_title') - - json_album = json_tralbum.get('packages') - if json_album: - json_album = json_album[0] - album_publish_date = json_album.get('album_publish_date') - album_release_date = json_album.get('album_release_date') - else: - album_publish_date = None - album_release_date = json_tralbum.get('album_release_date') - - timestamp = unified_timestamp(json_tralbum.get('current', {}).get('publish_date') or album_publish_date) - release_date = unified_strdate(album_release_date) - - download_link = self._search_regex( - r'freeDownloadPage(?:["\']|"):\s*(["\']|")(?P<url>(?:(?!\1).)+)\1', webpage, - 'download link', default=None, group='url') + tralbum = self._extract_data_attr(webpage, title) + thumbnail = self._og_search_thumbnail(webpage) + + track_id = None + track = None + track_number = None + duration = None + + formats = [] + track_info = try_get(tralbum, lambda x: x['trackinfo'][0], dict) + if track_info: + file_ = track_info.get('file') + if isinstance(file_, dict): + for format_id, format_url in file_.items(): + if not url_or_none(format_url): + continue + ext, abr_str = format_id.split('-', 1) + formats.append({ + 'format_id': format_id, + 'url': self._proto_relative_url(format_url, 'http:'), + 'ext': ext, + 'vcodec': 'none', + 'acodec': ext, + 'abr': int_or_none(abr_str), + }) + track = track_info.get('title') + track_id = str_or_none( + track_info.get('track_id') or track_info.get('id')) + track_number = int_or_none(track_info.get('track_num')) + duration = float_or_none(track_info.get('duration')) + + embed = self._extract_data_attr(webpage, title, 'embed', False) + current = tralbum.get('current') or {} + artist = embed.get('artist') or current.get('artist') or tralbum.get('artist') + timestamp = unified_timestamp( + current.get('publish_date') or tralbum.get('album_publish_date')) + + download_link = tralbum.get('freeDownloadPage') if download_link: - track_id = self._search_regex( - r'\?id=(?P<id>\d+)&', - download_link, 'track id') + track_id = compat_str(tralbum['id']) download_webpage = self._download_webpage( download_link, track_id, 'Downloading free downloads page') - blob = self._parse_json( - self._search_regex( - r'data-blob=(["\'])(?P<blob>{.+?})\1', download_webpage, - 'blob', group='blob'), - track_id, transform_source=unescapeHTML) + blob = self._extract_data_attr(download_webpage, track_id, 'blob') info = try_get( blob, (lambda x: x['digital_items'][0], @@ -173,6 +139,8 @@ class BandcampIE(BandcampBaseIE): if info: downloads = info.get('downloads') if isinstance(downloads, dict): + if not track: + track = info.get('title') if not artist: artist = info.get('artist') if not thumbnail: @@ -206,7 +174,7 @@ class BandcampIE(BandcampBaseIE): retry_url = url_or_none(stat.get('retry_url')) if not retry_url: continue - track['formats'].append({ + formats.append({ 'url': self._proto_relative_url(retry_url, 'http:'), 'ext': download_formats.get(format_id), 'format_id': format_id, @@ -215,30 +183,34 @@ class BandcampIE(BandcampBaseIE): 'vcodec': 'none', }) - self._sort_formats(track['formats']) + self._sort_formats(formats) - title = '%s - %s' % (artist, track.get('title')) if artist else track.get('title') + title = '%s - %s' % (artist, track) if artist else track + + if not duration: + duration = float_or_none(self._html_search_meta( + 'duration', webpage, default=None)) return { - 'album': album_title, - 'artist': artist, - 'duration': track['duration'], - 'formats': track['formats'], - 'id': track['id'], - 'release_date': release_date, + 'id': track_id, + 'title': title, 'thumbnail': thumbnail, + 'uploader': artist, 'timestamp': timestamp, - 'title': title, - 'track': track['title'], - 'track_id': track['id'], - 'track_number': track['number'], - 'uploader': artist + 'release_date': unified_strdate(tralbum.get('album_release_date')), + 'duration': duration, + 'track': track, + 'track_number': track_number, + 'track_id': track_id, + 'artist': artist, + 'album': embed.get('album_title'), + 'formats': formats, } -class BandcampAlbumIE(BandcampBaseIE): +class BandcampAlbumIE(BandcampIE): IE_NAME = 'Bandcamp:album' - _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^/?#&]+))?' + _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<id>[^/?#&]+))?' _TESTS = [{ 'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', @@ -248,7 +220,10 @@ class BandcampAlbumIE(BandcampBaseIE): 'info_dict': { 'id': '1353101989', 'ext': 'mp3', - 'title': 'Intro', + 'title': 'Blazo - Intro', + 'timestamp': 1311756226, + 'upload_date': '20110727', + 'uploader': 'Blazo', } }, { @@ -256,7 +231,10 @@ class BandcampAlbumIE(BandcampBaseIE): 'info_dict': { 'id': '38097443', 'ext': 'mp3', - 'title': 'Kero One - Keep It Alive (Blazo remix)', + 'title': 'Blazo - Kero One - Keep It Alive (Blazo remix)', + 'timestamp': 1311757238, + 'upload_date': '20110727', + 'uploader': 'Blazo', } }, ], @@ -292,6 +270,7 @@ class BandcampAlbumIE(BandcampBaseIE): 'title': '"Entropy" EP', 'uploader_id': 'jstrecords', 'id': 'entropy-ep', + 'description': 'md5:0ff22959c943622972596062f2f366a5', }, 'playlist_mincount': 3, }, { @@ -301,6 +280,7 @@ class BandcampAlbumIE(BandcampBaseIE): 'id': 'we-are-the-plague', 'title': 'WE ARE THE PLAGUE', 'uploader_id': 'insulters', + 'description': 'md5:b3cf845ee41b2b1141dc7bde9237255f', }, 'playlist_count': 2, }] @@ -312,41 +292,34 @@ class BandcampAlbumIE(BandcampBaseIE): else super(BandcampAlbumIE, cls).suitable(url)) def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - uploader_id = mobj.group('subdomain') - album_id = mobj.group('album_id') + uploader_id, album_id = re.match(self._VALID_URL, url).groups() playlist_id = album_id or uploader_id webpage = self._download_webpage(url, playlist_id) - - json_tralbum = self._extract_json_from_html_data_attribute(webpage, "tralbum", playlist_id) - json_embed = self._extract_json_from_html_data_attribute(webpage, "embed", playlist_id) - - json_tracks = json_tralbum.get('trackinfo') - if not json_tracks: - raise ExtractorError('Could not extract album tracks') - - album_title = json_embed.get('album_title') - + tralbum = self._extract_data_attr(webpage, playlist_id) + track_info = tralbum.get('trackinfo') + if not track_info: + raise ExtractorError('The page doesn\'t contain any tracks') # Only tracks with duration info have songs - tracks = [self._parse_json_track(track) for track in json_tracks] entries = [ self.url_result( - compat_urlparse.urljoin(url, track['title_link']), - ie=BandcampIE.ie_key(), video_id=track['id'], - video_title=track['title']) - for track in tracks - if track.get('duration')] + urljoin(url, t['title_link']), BandcampIE.ie_key(), + str_or_none(t.get('track_id') or t.get('id')), t.get('title')) + for t in track_info + if t.get('duration')] + + current = tralbum.get('current') or {} return { '_type': 'playlist', 'uploader_id': uploader_id, 'id': playlist_id, - 'title': album_title, - 'entries': entries + 'title': current.get('title'), + 'description': current.get('about'), + 'entries': entries, } -class BandcampWeeklyIE(InfoExtractor): +class BandcampWeeklyIE(BandcampIE): IE_NAME = 'Bandcamp:weekly' _VALID_URL = r'https?://(?:www\.)?bandcamp\.com/?\?(?:.*?&)?show=(?P<id>\d+)' _TESTS = [{ @@ -361,29 +334,23 @@ class BandcampWeeklyIE(InfoExtractor): 'release_date': '20170404', 'series': 'Bandcamp Weekly', 'episode': 'Magic Moments', - 'episode_number': 208, 'episode_id': '224', - } + }, + 'params': { + 'format': 'opus-lo', + }, }, { 'url': 'https://bandcamp.com/?blah/blah@&show=228', 'only_matching': True }] def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - - blob = self._parse_json( - self._search_regex( - r'data-blob=(["\'])(?P<blob>{.+?})\1', webpage, - 'blob', group='blob'), - video_id, transform_source=unescapeHTML) + show_id = self._match_id(url) + webpage = self._download_webpage(url, show_id) - show = blob['bcw_show'] + blob = self._extract_data_attr(webpage, show_id, 'blob') - # This is desired because any invalid show id redirects to `bandcamp.com` - # which happens to expose the latest Bandcamp Weekly episode. - show_id = int_or_none(show.get('show_id')) or int_or_none(video_id) + show = blob['bcw_data'][show_id] formats = [] for format_id, format_url in show['audio_stream'].items(): @@ -408,20 +375,8 @@ class BandcampWeeklyIE(InfoExtractor): if subtitle: title += ' - %s' % subtitle - episode_number = None - seq = blob.get('bcw_seq') - - if seq and isinstance(seq, list): - try: - episode_number = next( - int_or_none(e.get('episode_number')) - for e in seq - if isinstance(e, dict) and int_or_none(e.get('id')) == show_id) - except StopIteration: - pass - return { - 'id': video_id, + 'id': show_id, 'title': title, 'description': show.get('desc') or show.get('short_desc'), 'duration': float_or_none(show.get('audio_duration')), @@ -429,7 +384,6 @@ class BandcampWeeklyIE(InfoExtractor): 'release_date': unified_strdate(show.get('published_date')), 'series': 'Bandcamp Weekly', 'episode': show.get('subtitle'), - 'episode_number': episode_number, - 'episode_id': compat_str(video_id), + 'episode_id': show_id, 'formats': formats } |